CalvinBackup/llmsecops-research
1
0
Fork 0
mirror of https://github.com/lightbroker/llmsecops-research.git synced 2026-03-19 16:54:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #25 from lightbroker/model-support-expansion

Browse Source 
base classes
This commit is contained in:
Adam Wilson
2025-08-18 20:25:03 -06:00
committed by GitHub
parent 42054a08dd 42e4962063
commit 196148ffe0
16 changed files with 167 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
.github/workflows/test_04.abstract_base.yml vendored Normal file
View File

@@ -0,0 +1,72 @@
name: 'Reusable Test Runner | RAG + CoT | Generic'
on:
workflow_call:
inputs:
batch_offset:
description: 'Starting prompt index offset'
required: true
type: number
range_name:
description: 'Human readable range name (e.g., "1-20")'
required: true
type: string
test_file_path:
description: 'Path to the test file to run'
required: true
type: string
model_display_name:
description: 'Human readable model name for workflow title'
required: true
type: string
batch_size:
description: 'Number of prompts per batch'
required: false
type: number
default: 2
jobs:
test:
runs-on: ubuntu-latest
strategy:
# Always 10 batches per workflow
matrix:
batch: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
fail-fast: false
steps:
- name: 'checkout'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- name: 'set up Python'
uses: actions/setup-python@v3
with:
python-version: '3.12'
- name: 'set up Python dependencies'
shell: bash
run: pip install -r ${{ github.workspace }}/requirements.txt
- name: 'run text generation tests - ${{ inputs.model_display_name }} - range ${{ inputs.range_name }} batch ${{ matrix.batch }}'
shell: bash
env:
PROMPT_BATCH: ${{ matrix.batch }}
BATCH_SIZE: ${{ inputs.batch_size }}
BATCH_OFFSET: ${{ inputs.batch_offset }}
run: pytest ${{ inputs.test_file_path }} -s --disable-warnings
- name: Check for changes
id: verify-changed-files
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "changed=false" >> $GITHUB_OUTPUT
fi
- name: Commit and push changes
if: steps.verify-changed-files.outputs.changed == 'true'
run: |
git config --local user.email "42450907+lightbroker@users.noreply.github.com"
git config --local user.name "Adam Wilson"
git add .
git commit -m "Auto-generated files from ${{ inputs.model_display_name }} range ${{ inputs.range_name }} batch ${{ matrix.batch }} [skip ci]"
git push

11
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.01-20.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
name: '#4 (1-20) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:
jobs:
test-prompts-1-20:
uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
with:
batch_offset: 0
range_name: "1-20"
batch_size: 2

11
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.21-40.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
name: '#4 (21-40) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:
jobs:
test-prompts-21-40:
uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
with:
batch_offset: 20
range_name: "21-40"
batch_size: 2

11
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.41-60.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
name: '#4 (41-60) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:
jobs:
test-prompts-41-60:
uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
with:
batch_offset: 40
range_name: "41-60"
batch_size: 2

11
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.61-80.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
name: '#4 (61-80) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:
jobs:
test-prompts-61-80:
uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
with:
batch_offset: 60
range_name: "61-80"
batch_size: 2

11
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.81-100.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
name: '#4 (81-100) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:
jobs:
test-prompts-81-100:
uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
with:
batch_offset: 80
range_name: "81-100"
batch_size: 2

27
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml vendored Normal file
View File

@@ -0,0 +1,27 @@
name: 'Reusable Test #4 | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
on:
workflow_call:
inputs:
batch_offset:
description: 'Starting prompt index offset'
required: true
type: number
range_name:
description: 'Human readable range name (e.g., "1-20")'
required: true
type: string
batch_size:
description: 'Number of prompts per batch'
required: false
type: number
default: 2
jobs:
test:
uses: ./.github/workflows/test_04.abstract_base.yml
with:
batch_offset: ${{ inputs.batch_offset }}
range_name: ${{ inputs.range_name }}
batch_size: ${{ inputs.batch_size }}
test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.py
model_display_name: meta-llama/Llama-3.2-3B-Instruct

42
.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.yml vendored
View File

@@ -1,42 +0,0 @@
name: 'Test #4 | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
on:
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: 'checkout'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- name: 'set up Python'
uses: actions/setup-python@v3
with:
python-version: '3.12'
- name: 'set up Python dependencies'
shell: bash
run: pip install -r ${{ github.workspace }}/requirements.txt
- name: 'run text generation tests'
shell: bash
run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.py -s --disable-warnings
- name: Check for changes
id: verify-changed-files
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "changed=false" >> $GITHUB_OUTPUT
fi
- name: Commit and push changes
if: steps.verify-changed-files.outputs.changed == 'true'
run: |
git config --local user.email "42450907+lightbroker@users.noreply.github.com"
git config --local user.name "Adam Wilson"
git add .
git commit -m "Auto-generated files from workflow [skip ci]"
git push

2
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.01-20.yml vendored
View File

@@ -1,4 +1,4 @@
name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (1-20)'
name: '#4 (1-20) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:

2
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.21-40.yml vendored
View File

@@ -1,4 +1,4 @@
name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (21-40)'
name: '#4 (21-40) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:

2
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.41-60.yml vendored
View File

@@ -1,4 +1,4 @@
name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (41-60)'
name: '#4 (41-60) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:

2
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.61-80.yml vendored
View File

@@ -1,4 +1,4 @@
name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (61-80)'
name: '#4 (61-80) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:

2
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.81-100.yml vendored
View File

@@ -1,4 +1,4 @@
name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (81-100)'
name: '#4 (81-100) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
on:
workflow_dispatch:

46
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml vendored
View File

@@ -18,42 +18,10 @@ on:
jobs:
test:
runs-on: ubuntu-latest
strategy:
# Always 10 batches per workflow
matrix:
batch: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
fail-fast: false
steps:
- name: 'checkout'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- name: 'set up Python'
uses: actions/setup-python@v3
with:
python-version: '3.12'
- name: 'set up Python dependencies'
shell: bash
run: pip install -r ${{ github.workspace }}/requirements.txt
- name: 'run text generation tests - range ${{ inputs.range_name }} batch ${{ matrix.batch }}'
shell: bash
env:
PROMPT_BATCH: ${{ matrix.batch }}
BATCH_SIZE: ${{ inputs.batch_size }}
BATCH_OFFSET: ${{ inputs.batch_offset }}
run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.py -s --disable-warnings
- name: Check for changes
id: verify-changed-files
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "changed=false" >> $GITHUB_OUTPUT
fi
- name: Commit and push changes
if: steps.verify-changed-files.outputs.changed == 'true'
run: |
git config --local user.email "42450907+lightbroker@users.noreply.github.com"
git config --local user.name "Adam Wilson"
git add .
git commit -m "Auto-generated files from range ${{ inputs.range_name }} batch ${{ matrix.batch }} [skip ci]"
git push
uses: ./.github/workflows/test_04.abstract_base.yml
with:
batch_offset: ${{ inputs.batch_offset }}
range_name: ${{ inputs.range_name }}
batch_size: ${{ inputs.batch_size }}
test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.py
model_display_name: microsoft/Phi-3-mini-4k-instruct

44
.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.yml vendored
View File

@@ -1,44 +0,0 @@
name: 'Test #4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct | Prompts 1-20'
on:
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
strategy:
# Matrix for batches 1-10, each testing 2 prompts (prompts 1-20 total)
matrix:
batch: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
fail-fast: false
steps:
- name: 'checkout'
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- name: 'set up Python'
uses: actions/setup-python@v3
with:
python-version: '3.12'
- name: 'set up Python dependencies'
shell: bash
run: pip install -r ${{ github.workspace }}/requirements.txt
- name: 'run text generation tests - batch ${{ matrix.batch }}'
shell: bash
env:
PROMPT_BATCH: ${{ matrix.batch }}
BATCH_SIZE: 2
BATCH_OFFSET: 0 # Prompts 1-20 (offset 0)
run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.py -s --disable-warnings
- name: Check for changes
id: verify-changed-files
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "changed=false" >> $GITHUB_OUTPUT
fi
- name: Commit and push changes (batch ${{ matrix.batch }})
if: steps.verify-changed-files.outputs.changed == 'true'
run: |
git config --local user.email "42450907+lightbroker@users.noreply.github.com"
git config --local user.name "Adam Wilson"
git add .
git commit -m "Auto-generated files from workflow batch ${{ matrix.batch }} (prompts 1-20) [skip ci]"
git push

2
src/text_generation/services/logging/test_run_logging_service.py
View File

@@ -22,7 +22,7 @@ class TestRunLoggingService(AbstractTestRunLoggingService):
):
self._lock = threading.Lock()
timestamp = calendar.timegm(time.gmtime())
self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/{start}_{end}/test_{test_id}_logs_{timestamp}.json"
self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/{start}_{end}/test_{str(test_id).lower()}_logs_{timestamp}.json"
self._ensure_log_file_exists()
def _ensure_log_file_exists(self):