llmsecops-research/.github/workflows/archived/llmsecops-cicd.llm_rag.yml

name: '[Deprecated] LLM Prompt Testing (WSGI)'

on:
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: 'checkout'
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      
      - name: 'set up git LFS'
        run: git lfs install
      
      - name: 'set up Python'
        uses: actions/setup-python@v3
        with:
          python-version: '3.12'
      
      - name: 'set up Python dependencies'
        run: |
          pip install -r ${{ github.workspace }}/requirements.txt
      
      - name: 'set up Microsoft Phi-3 Mini 4k LLM from HuggingFace'
        id: setup_llm
        run: |
          pip install huggingface-hub[cli]
          huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx \
            --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* \
            --local-dir ${{ github.workspace }}/src/llm
        continue-on-error: false
      
      - name: 'set up Garak'
        run: |
          pip install garak
        continue-on-error: false
      
      - name: 'start HTTP server'
        id: start_server
        run: |
          nohup python -m src.api.server > server.log 2>&1 &
          server_pid=$!
          echo "Server PID: $server_pid"
          echo "server_pid=$server_pid" >> $GITHUB_ENV
          
          # Wait for server to start and verify it's running
          max_retries=30
          retry_count=0
          server_ready=false
          
          while [ $retry_count -lt $max_retries ] && [ "$server_ready" = false ]; do
            echo "Waiting for server to start (attempt $retry_count/$max_retries)..."
            if curl -s -o /dev/null -w "%{http_code}" localhost:9999 > /dev/null 2>&1; then
              server_ready=true
              echo "Server is running"
            else
              sleep 2
              retry_count=$((retry_count + 1))
            fi
          done
          
          if [ "$server_ready" = false ]; then
            echo "::error::Server failed to start after $max_retries attempts"
            echo "=== Server Log (last 50 lines) ==="
            tail -n 50 server.log || true
            exit 1
          fi
      
      - name: 'Test server with curl and run garak'
        id: run_tests
        run: |
          # Test curl with detailed error reporting
          # curl_output=$(curl -X POST -i localhost:9999/api/completions -d '{ "prompt": "describe a random planet in our solar system in 10 words or less" }' --connect-timeout 10 -v 2>&1) || true
          # echo "$curl_output"
          
          garak -v \
            --config ${{ github.workspace }}/src/tools/garak.config.yml \
            --generator_option_file ${{ github.workspace }}/src/tools/garak.rest.llm-rag.json \
            --model_type=rest \
            --parallel_attempts 32
          garak_exit_code=$?
          echo "garak exit code: $garak_exit_code"
          
          # Store exit code for later use
          echo "garak_exit_code=$garak_exit_code" >> $GITHUB_ENV
        continue-on-error: true
      
      - name: 'Collect and display server logs'
        if: always()
        run: |
          echo "::group::Server Log"
          cat server.log || true
          echo "::endgroup::"
          
          # Check if server process is still running and kill it
          if [ -n "$server_pid" ]; then
            echo "Stopping server process (PID: $server_pid)..."
            kill -9 $server_pid 2>/dev/null || true
          fi
          
          # Create a summary of the workflow
          echo "# LLM Prompt Testing Workflow Summary" > $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # Add curl test results to summary
          echo "## Curl Test Results" >> $GITHUB_STEP_SUMMARY
          if [[ "${{ steps.run_tests.outcome }}" == "success" ]]; then
            echo "✅ Curl request test succeeded" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ Curl request test failed" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # Add Garak results to summary
          echo "## Garak Test Results" >> $GITHUB_STEP_SUMMARY
          if [[ "$garak_exit_code" == "0" ]]; then
            echo "✅ Garak tests succeeded" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ Garak tests failed with exit code $garak_exit_code" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # Add server log summary
          echo "## Server Log Summary" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          tail -n 30 server.log >> $GITHUB_STEP_SUMMARY || echo "No server log available" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
      
      - name: 'Collect system diagnostics'
        if: always()
        run: |
          # Create diagnostics file
          echo "::group::System Diagnostics"
          diagnostics_file="system_diagnostics.txt"
          echo "=== System Information ===" > $diagnostics_file
          uname -a >> $diagnostics_file
          echo "" >> $diagnostics_file
          
          echo "=== Network Status ===" >> $diagnostics_file
          echo "Checking port 9999:" >> $diagnostics_file
          ss -tulpn | grep 9999 >> $diagnostics_file || echo "No process found on port 9999" >> $diagnostics_file
          echo "" >> $diagnostics_file
          
          echo "=== Process Status ===" >> $diagnostics_file
          ps aux | grep python >> $diagnostics_file
          echo "" >> $diagnostics_file
          
          echo "=== Memory Usage ===" >> $diagnostics_file
          free -h >> $diagnostics_file
          echo "" >> $diagnostics_file
          
          cat $diagnostics_file
          echo "::endgroup::"
      
      - name: 'Upload logs as artifacts'
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: workflow-logs
          path: |
            server.log
            system_diagnostics.txt
            ${{ github.workspace }}/src/tools/garak.config.yml
            ${{ github.workspace }}/src/tools/garak.rest.llm.json
          retention-days: 7
      
      # Final status check to fail the workflow if tests failed
      - name: 'Check final status'
        if: always()
        run: |
          if [[ "${{ steps.run_tests.outcome }}" != "success" || "$garak_exit_code" != "0" ]]; then
            echo "::error::Tests failed - check logs and summary for details"
            exit 1
          fi