From d238e69f6431ff2e6c4decb317d7a903a4ae67bd Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 17:28:32 -0600 Subject: [PATCH 1/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index d6253d343..20a199f3a 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -28,6 +28,10 @@ jobs: run: | pip install garak - - name: 'run HTTP server and call REST API' + - name: 'Garak test probe' run: | - garak -v --model_type test.Blank --probes test.Test \ No newline at end of file + garak -v --model_type test.Blank --probes test.Test --report_dir ${{ github.workspace }}/reports + + - name: 'display report' + run: | + ls ${{ github.workspace }}/reports -al \ No newline at end of file From f58056b2cc9330c4b6a4607964e935c9ce6e1b36 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 17:34:22 -0600 Subject: [PATCH 2/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index 20a199f3a..b9534edd2 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -30,7 +30,7 @@ jobs: - name: 'Garak test probe' run: | - garak -v --model_type test.Blank --probes test.Test --report_dir ${{ github.workspace }}/reports + python -m garak -v --model_type test.Blank --probes test.Test --report_dir ${{ github.workspace }}/reports - name: 'display report' run: | From 5000b5e0319e10d108e242e209eb677cad2a8c62 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 17:39:01 -0600 Subject: [PATCH 3/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index b9534edd2..100850ff9 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -30,7 +30,7 @@ jobs: - name: 'Garak test probe' run: | - python -m garak -v --model_type test.Blank --probes test.Test --report_dir ${{ github.workspace }}/reports + python -m garak --model_type test.Blank --probes test.Test --report_dir ${{ github.workspace }}/reports - name: 'display report' run: | From ca823b70e64968b1fbf030b66deeebc72ceac016 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 17:51:17 -0600 Subject: [PATCH 4/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 2 +- tests/tools/garak.config.test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index 100850ff9..eb1df191c 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -30,7 +30,7 @@ jobs: - name: 'Garak test probe' run: | - python -m garak --model_type test.Blank --probes test.Test --report_dir ${{ github.workspace }}/reports + python -m garak --model_type test.Blank --probes test.Test - name: 'display report' run: | diff --git a/tests/tools/garak.config.test.yml b/tests/tools/garak.config.test.yml index ae210483c..befc58e06 100644 --- a/tests/tools/garak.config.test.yml +++ b/tests/tools/garak.config.test.yml @@ -2,4 +2,4 @@ run: generations: 1 plugins: - probe_spec: promptinject.HijackHateHumansMini \ No newline at end of file + probe_spec: test.Test \ No newline at end of file From 2deae8e52ed6436e278edbe780051883d96bb7a3 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 18:01:33 -0600 Subject: [PATCH 5/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index eb1df191c..e0d13bca9 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -31,7 +31,13 @@ jobs: - name: 'Garak test probe' run: | python -m garak --model_type test.Blank --probes test.Test + - name: 'display report' run: | - ls ${{ github.workspace }}/reports -al \ No newline at end of file + ls /home/runner/.local/share/garak/garak_runs/ -al + echo + cat /home/runner/.local/share/garak/garak_runs/garak.*.jsonl + echo + echo + cat /home/runner/.local/share/garak/garak_runs/garak.*.html From 1ed07dd483b5ad68acead54f17fb7cb5ba535aa8 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 18:15:55 -0600 Subject: [PATCH 6/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index e0d13bca9..3f33883d8 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -41,3 +41,8 @@ jobs: echo echo cat /home/runner/.local/share/garak/garak_runs/garak.*.html + + - uses: actions/upload-artifact@v4 + with: + name: 'garak_report' + path: /home/runner/.local/share/garak/garak_runs/garak.*.html \ No newline at end of file From e2813ad6c23c567f6cc1a4658ce4747d20c0d68f Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 3 May 2025 18:22:49 -0600 Subject: [PATCH 7/7] reporting test --- .github/workflows/llmsecops-cicd.test.yml | 2 +- .github/workflows/llmsecops-cicd.yml | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/llmsecops-cicd.test.yml b/.github/workflows/llmsecops-cicd.test.yml index 3f33883d8..8e4fe1f58 100644 --- a/.github/workflows/llmsecops-cicd.test.yml +++ b/.github/workflows/llmsecops-cicd.test.yml @@ -42,7 +42,7 @@ jobs: echo cat /home/runner/.local/share/garak/garak_runs/garak.*.html - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: 'garak_report' path: /home/runner/.local/share/garak/garak_runs/garak.*.html \ No newline at end of file diff --git a/.github/workflows/llmsecops-cicd.yml b/.github/workflows/llmsecops-cicd.yml index 105d5e422..88e760951 100644 --- a/.github/workflows/llmsecops-cicd.yml +++ b/.github/workflows/llmsecops-cicd.yml @@ -39,12 +39,15 @@ jobs: curl -X POST -i localhost:9999 -d '{ "prompt": "describe a random planet in our solar system in 10 words or less" }' || true echo - garak -v --model_type test.Blank --probes test.Test - garak -v \ --config ${{ github.workspace }}/tests/tools/garak.config.yml \ --generator_option_file ${{ github.workspace }}/tests/tools/garak.rest.json \ --model_type=rest \ - --parallel_attempts 16 + --parallel_attempts 32 - cat server.log \ No newline at end of file + cat server.log + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: 'garak_report' + path: /home/runner/.local/share/garak/garak_runs/garak.*.html \ No newline at end of file