diff --git a/.github/workflows/llmsecops-cicd.yml b/.github/workflows/llmsecops-cicd.yml index 85b3d875d..fe8f42d0f 100644 --- a/.github/workflows/llmsecops-cicd.yml +++ b/.github/workflows/llmsecops-cicd.yml @@ -6,49 +6,170 @@ on: jobs: build: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - - name: 'set up git LFS' - run: git lfs install - - - name: 'set up Python' - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - - name: 'set up Python dependencies' - run: | - pip install -r ${{ github.workspace }}/requirements.txt - - - name: 'set up Microsoft Phi-3 Mini 4k LLM from HuggingFace' - run: | - pip install huggingface-hub[cli] - huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx \ - --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* \ - --local-dir ${{ github.workspace }}/src/llm - - - name: 'set up Garak' - run: | - pip install garak - - - name: 'run HTTP server and call REST API' - run: | - nohup python -m src.api.server > server.log 2>&1 & - sleep 5 - curl -X POST -i localhost:9999 -d '{ "prompt": "describe a random planet in our solar system in 10 words or less" }' || true - echo - - garak -v \ - --config ${{ github.workspace }}/src/tools/garak.config.yml \ - --generator_option_file ${{ github.workspace }}/src/tools/garak.rest.llm.json \ - --model_type=rest \ - --parallel_attempts 16 - - cat server.log - - # - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - # with: - # name: 'garak_report' - # path: /home/runner/.local/share/garak/garak_runs/garak.*.html \ No newline at end of file + - name: 'checkout' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: 'set up git LFS' + run: git lfs install + + - name: 'set up Python' + uses: actions/setup-python@v3 + with: + python-version: '3.12' + + - name: 'set up Python dependencies' + run: | + pip install -r ${{ github.workspace }}/requirements.txt + + - name: 'set up Microsoft Phi-3 Mini 4k LLM from HuggingFace' + id: setup_llm + run: | + pip install huggingface-hub[cli] + huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx \ + --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* \ + --local-dir ${{ github.workspace }}/src/llm + continue-on-error: false + + - name: 'set up Garak' + run: | + pip install garak + continue-on-error: false + + - name: 'start HTTP server' + id: start_server + run: | + nohup python -m src.api.server > server.log 2>&1 & + server_pid=$! + echo "Server PID: $server_pid" + echo "server_pid=$server_pid" >> $GITHUB_ENV + + # Wait for server to start and verify it's running + max_retries=30 + retry_count=0 + server_ready=false + + while [ $retry_count -lt $max_retries ] && [ "$server_ready" = false ]; do + echo "Waiting for server to start (attempt $retry_count/$max_retries)..." + if curl -s -o /dev/null -w "%{http_code}" localhost:9999 > /dev/null 2>&1; then + server_ready=true + echo "Server is running" + else + sleep 2 + retry_count=$((retry_count + 1)) + fi + done + + if [ "$server_ready" = false ]; then + echo "::error::Server failed to start after $max_retries attempts" + echo "=== Server Log (last 50 lines) ===" + tail -n 50 server.log || true + exit 1 + fi + + - name: 'Test server with curl and run garak' + id: run_tests + run: | + # Test curl with detailed error reporting + curl_output=$(curl -X POST -i localhost:9999/api/conversations -d '{ "prompt": "describe a random planet in our solar system in 10 words or less" }' --connect-timeout 10 -v 2>&1) || true + echo "$curl_output" + + garak -v \ + --config ${{ github.workspace }}/src/tools/garak.config.yml \ + --generator_option_file ${{ github.workspace }}/src/tools/garak.rest.llm.json \ + --model_type=rest \ + --parallel_attempts 16 + garak_exit_code=$? + echo "garak exit code: $garak_exit_code" + + # Store exit code for later use + echo "garak_exit_code=$garak_exit_code" >> $GITHUB_ENV + continue-on-error: true + + - name: 'Collect and display server logs' + if: always() + run: | + echo "::group::Server Log" + cat server.log || true + echo "::endgroup::" + + # Check if server process is still running and kill it + if [ -n "$server_pid" ]; then + echo "Stopping server process (PID: $server_pid)..." + kill -9 $server_pid 2>/dev/null || true + fi + + # Create a summary of the workflow + echo "# LLM Prompt Testing Workflow Summary" > $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Add curl test results to summary + echo "## Curl Test Results" >> $GITHUB_STEP_SUMMARY + if [[ "${{ steps.run_tests.outcome }}" == "success" ]]; then + echo "✅ Curl request test succeeded" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Curl request test failed" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + # Add Garak results to summary + echo "## Garak Test Results" >> $GITHUB_STEP_SUMMARY + if [[ "$garak_exit_code" == "0" ]]; then + echo "✅ Garak tests succeeded" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Garak tests failed with exit code $garak_exit_code" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + # Add server log summary + echo "## Server Log Summary" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -n 30 server.log >> $GITHUB_STEP_SUMMARY || echo "No server log available" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: 'Collect system diagnostics' + if: always() + run: | + # Create diagnostics file + echo "::group::System Diagnostics" + diagnostics_file="system_diagnostics.txt" + echo "=== System Information ===" > $diagnostics_file + uname -a >> $diagnostics_file + echo "" >> $diagnostics_file + + echo "=== Network Status ===" >> $diagnostics_file + echo "Checking port 9999:" >> $diagnostics_file + ss -tulpn | grep 9999 >> $diagnostics_file || echo "No process found on port 9999" >> $diagnostics_file + echo "" >> $diagnostics_file + + echo "=== Process Status ===" >> $diagnostics_file + ps aux | grep python >> $diagnostics_file + echo "" >> $diagnostics_file + + echo "=== Memory Usage ===" >> $diagnostics_file + free -h >> $diagnostics_file + echo "" >> $diagnostics_file + + cat $diagnostics_file + echo "::endgroup::" + + - name: 'Upload logs as artifacts' + if: always() + uses: actions/upload-artifact@v3 + with: + name: workflow-logs + path: | + server.log + system_diagnostics.txt + ${{ github.workspace }}/src/tools/garak.config.yml + ${{ github.workspace }}/src/tools/garak.rest.llm.json + retention-days: 7 + + # Final status check to fail the workflow if tests failed + - name: 'Check final status' + if: always() + run: | + if [[ "${{ steps.run_tests.outcome }}" != "success" || "$garak_exit_code" != "0" ]]; then + echo "::error::Tests failed - check logs and summary for details" + exit 1 + fi \ No newline at end of file