Merge pull request #5 from lightbroker/garak-setup

Garak setup

.github/workflows/llmsecops-cicd.test.yml

@@ -0,0 +1,33 @@
+name: 'LLM Prompt Testing (Garak test.Test probe)'
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683    
+
+    # - name: 'set up git LFS'
+    #   run: git lfs install
+
+    - name: 'set up Python'
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.12'
+
+    # - name: 'set up Microsoft Phi-3 Mini 4k LLM from HuggingFace'
+    #   run: |
+    #     pip install huggingface-hub[cli]
+    #     huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir ${{ github.workspace }}/tests/llm
+    #     pip install onnxruntime-genai
+
+    - name: 'set up Garak'
+      run: |
+        pip install garak
+
+    - name: 'run HTTP server and call REST API'
+      run: |
+        garak -v --model_type test.Blank --probes test.Test

.github/workflows/llmsecops-cicd.yml

@@ -28,9 +28,23 @@ jobs:
         huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir ${{ github.workspace }}/tests/llm
         pip install onnxruntime-genai
 
+    - name: 'set up Garak'
+      run: |
+        pip install garak
+
     - name: 'run HTTP server and call REST API'
       run: |
         nohup python -m tests.api.server > server.log 2>&1 &
         sleep 2
         curl -X POST -i localhost:9999 -d '{ "prompt": "describe a random planet in our solar system in 10 words or less" }' || true
+        echo
+        
+        garak -v --model_type test.Blank --probes test.Test
+
+        garak -v \
+          --config ${{ github.workspace }}/tests/tools/garak.config.yml \
+          --generator_option_file ${{ github.workspace }}/tests/tools/garak.rest.json \
+          --model_type=rest \
+          --parallel_attempts 16
+        
         cat server.log

tests/tools/garak.config.test.yml

@@ -0,0 +1,5 @@
+run:
+    generations: 1
+
+plugins:
+    probe_spec: promptinject.HijackHateHumansMini

tests/tools/garak.config.yml

@@ -0,0 +1,5 @@
+run:
+    generations: 1
+
+plugins:
+    probe_spec: promptinject.HijackHateHumansMini

tests/tools/garak.rest.json

@@ -0,0 +1,19 @@
+{
+    "rest": {
+        "RestGenerator": {
+            "name": "Microsoft ONNX Phi-3 LLM/GenAI REST API Service",
+            "uri": "http://localhost:9999/",
+            "method": "post",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "req_template_json_object": {
+                "prompt": "$INPUT"
+            },
+            "response_json": true,
+            "response_json_field": "response",
+            "request_timeout": 600,
+            "verify_ssl": false
+        }
+    }
+}