mirror of
https://github.com/FuzzingLabs/fuzzforge_ai.git
synced 2026-02-13 03:52:45 +00:00
Compare commits
12 Commits
chore-code
...
test/autom
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bdc0aaa347 | ||
|
|
2bd0657d01 | ||
|
|
3fdbfcc6fd | ||
|
|
2d045d37f2 | ||
|
|
5bf481aee6 | ||
|
|
e0948533c0 | ||
|
|
52f168e2c2 | ||
|
|
ddc6f163f7 | ||
|
|
4c49d49cc8 | ||
|
|
853a8be8f3 | ||
|
|
3a16a802eb | ||
|
|
02b877d23d |
177
.github/test-matrix.yaml
vendored
Normal file
177
.github/test-matrix.yaml
vendored
Normal file
@@ -0,0 +1,177 @@
|
||||
# Test Matrix Configuration for Automated Workflow Testing
|
||||
#
|
||||
# This file defines which workflows to test, their required workers,
|
||||
# test projects, parameters, and expected outcomes.
|
||||
#
|
||||
# Excluded workflows:
|
||||
# - llm_analysis (requires LLM API keys)
|
||||
# - llm_secret_detection (requires LLM API keys)
|
||||
# - ossfuzz_campaign (requires OSS-Fuzz project configuration)
|
||||
|
||||
version: "1.0"
|
||||
|
||||
# Worker to Dockerfile mapping
|
||||
workers:
|
||||
android:
|
||||
dockerfiles:
|
||||
linux/amd64: "Dockerfile.amd64"
|
||||
linux/arm64: "Dockerfile.arm64"
|
||||
metadata: "workers/android/metadata.yaml"
|
||||
|
||||
python:
|
||||
dockerfiles:
|
||||
default: "Dockerfile"
|
||||
|
||||
rust:
|
||||
dockerfiles:
|
||||
default: "Dockerfile"
|
||||
|
||||
secrets:
|
||||
dockerfiles:
|
||||
default: "Dockerfile"
|
||||
|
||||
# Workflow test configurations
|
||||
workflows:
|
||||
# Android Static Analysis
|
||||
android_static_analysis:
|
||||
worker: android
|
||||
test_project: test_projects/android_test
|
||||
working_directory: test_projects/android_test
|
||||
parameters:
|
||||
apk_path: "BeetleBug.apk"
|
||||
timeout: 600
|
||||
platform_specific: true # Test on both amd64 and arm64
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [android, static-analysis, fast]
|
||||
|
||||
# Python SAST
|
||||
python_sast:
|
||||
worker: python
|
||||
test_project: test_projects/vulnerable_app
|
||||
working_directory: test_projects/vulnerable_app
|
||||
parameters: {}
|
||||
timeout: 180
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [python, sast, fast]
|
||||
|
||||
# Python Fuzzing (Atheris)
|
||||
atheris_fuzzing:
|
||||
worker: python
|
||||
test_project: test_projects/python_fuzz_waterfall
|
||||
working_directory: test_projects/python_fuzz_waterfall
|
||||
parameters:
|
||||
max_total_time: 30 # Short fuzzing run for testing
|
||||
artifact_prefix: "test-atheris"
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: false # May not find crashes in short run
|
||||
sarif_export: false
|
||||
tags: [python, fuzzing, slow]
|
||||
|
||||
# Rust Fuzzing (cargo-fuzz)
|
||||
cargo_fuzzing:
|
||||
worker: rust
|
||||
test_project: test_projects/rust_fuzz_test
|
||||
working_directory: test_projects/rust_fuzz_test
|
||||
parameters:
|
||||
max_total_time: 30 # Short fuzzing run for testing
|
||||
artifact_prefix: "test-cargo"
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: false # May not find crashes in short run
|
||||
sarif_export: false
|
||||
tags: [rust, fuzzing, slow]
|
||||
|
||||
# Secret Detection (combined)
|
||||
secret_detection:
|
||||
worker: secrets
|
||||
test_project: test_projects/secret_detection_benchmark
|
||||
working_directory: test_projects/secret_detection_benchmark
|
||||
parameters: {}
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [secrets, detection, fast]
|
||||
|
||||
# Gitleaks Detection
|
||||
gitleaks_detection:
|
||||
worker: secrets
|
||||
test_project: test_projects/secret_detection_benchmark
|
||||
working_directory: test_projects/secret_detection_benchmark
|
||||
parameters: {}
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [secrets, gitleaks, fast]
|
||||
|
||||
# TruffleHog Detection
|
||||
trufflehog_detection:
|
||||
worker: secrets
|
||||
test_project: test_projects/secret_detection_benchmark
|
||||
working_directory: test_projects/secret_detection_benchmark
|
||||
parameters: {}
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [secrets, trufflehog, fast]
|
||||
|
||||
# Security Assessment (composite workflow)
|
||||
security_assessment:
|
||||
worker: python # Uses multiple workers internally
|
||||
test_project: test_projects/vulnerable_app
|
||||
working_directory: test_projects/vulnerable_app
|
||||
parameters: {}
|
||||
timeout: 300
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [composite, security, slow]
|
||||
|
||||
# Test suites - groups of workflows for different scenarios
|
||||
test_suites:
|
||||
# Fast tests - run on every PR
|
||||
fast:
|
||||
workflows:
|
||||
- android_static_analysis
|
||||
- python_sast
|
||||
- gitleaks_detection
|
||||
- trufflehog_detection
|
||||
timeout: 900 # 15 minutes total
|
||||
|
||||
# Full tests - run on main/master
|
||||
full:
|
||||
workflows:
|
||||
- android_static_analysis
|
||||
- python_sast
|
||||
- atheris_fuzzing
|
||||
- cargo_fuzzing
|
||||
- secret_detection
|
||||
- gitleaks_detection
|
||||
- trufflehog_detection
|
||||
- security_assessment
|
||||
timeout: 1800 # 30 minutes total
|
||||
|
||||
# Platform-specific tests - test Dockerfile selection
|
||||
platform:
|
||||
workflows:
|
||||
- android_static_analysis
|
||||
- python_sast
|
||||
platforms:
|
||||
- linux/amd64
|
||||
- linux/arm64
|
||||
timeout: 600 # 10 minutes total
|
||||
375
.github/workflows/test-workflows.yml
vendored
Normal file
375
.github/workflows/test-workflows.yml
vendored
Normal file
@@ -0,0 +1,375 @@
|
||||
name: Workflow Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, dev, develop, test/** ]
|
||||
pull_request:
|
||||
branches: [ main, master, dev, develop ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_suite:
|
||||
description: 'Test suite to run'
|
||||
required: false
|
||||
default: 'fast'
|
||||
type: choice
|
||||
options:
|
||||
- fast
|
||||
- full
|
||||
- platform
|
||||
|
||||
jobs:
|
||||
#############################################################################
|
||||
# Platform Detection Unit Tests
|
||||
#############################################################################
|
||||
platform-detection-tests:
|
||||
name: Platform Detection Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pytest pytest-cov pyyaml
|
||||
# Install local monorepo dependencies first
|
||||
pip install -e ../sdk
|
||||
pip install -e ../ai
|
||||
# Then install CLI
|
||||
pip install -e .
|
||||
|
||||
- name: Run platform detection tests
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
pytest tests/test_platform_detection.py -v \
|
||||
--cov=src/fuzzforge_cli \
|
||||
--cov-report=term \
|
||||
--cov-report=xml
|
||||
|
||||
- name: Upload coverage
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
file: ./cli/coverage.xml
|
||||
flags: cli-platform-detection
|
||||
name: cli-platform-detection
|
||||
|
||||
#############################################################################
|
||||
# Fast Workflow Tests (AMD64 only)
|
||||
#############################################################################
|
||||
fast-workflow-tests:
|
||||
name: Fast Workflow Tests (AMD64)
|
||||
runs-on: ubuntu-latest
|
||||
needs: platform-detection-tests
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install FuzzForge CLI
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pyyaml # Required by test script
|
||||
# Install local monorepo dependencies first
|
||||
pip install -e ../sdk
|
||||
pip install -e ../ai
|
||||
# Then install CLI
|
||||
pip install -e .
|
||||
|
||||
- name: Copy environment template
|
||||
run: |
|
||||
mkdir -p volumes/env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
- name: Start FuzzForge services
|
||||
run: |
|
||||
docker compose up -d
|
||||
echo "⏳ Waiting for services to be ready..."
|
||||
sleep 30
|
||||
|
||||
# Wait for backend to be healthy
|
||||
max_wait=60
|
||||
waited=0
|
||||
while [ $waited -lt $max_wait ]; do
|
||||
if docker ps --filter "name=fuzzforge-backend" --format "{{.Status}}" | grep -q "healthy"; then
|
||||
echo "✅ Backend is healthy"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for backend... ($waited/$max_wait seconds)"
|
||||
sleep 5
|
||||
waited=$((waited + 5))
|
||||
done
|
||||
|
||||
- name: Initialize test projects
|
||||
run: |
|
||||
echo "Initializing test projects..."
|
||||
# Create minimal .fuzzforge directories for test projects
|
||||
for project in vulnerable_app android_test secret_detection_benchmark rust_test; do
|
||||
mkdir -p test_projects/$project/.fuzzforge
|
||||
cat > test_projects/$project/.fuzzforge/config.yaml <<EOF
|
||||
project:
|
||||
name: $project
|
||||
api_url: http://localhost:8000
|
||||
id: test-$(uuidgen | tr '[:upper:]' '[:lower:]' | tr -d '-' | head -c 16)
|
||||
EOF
|
||||
done
|
||||
|
||||
- name: Run fast workflow tests
|
||||
run: |
|
||||
python scripts/test_workflows.py --suite fast --skip-service-start
|
||||
timeout-minutes: 20
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Docker container status ==="
|
||||
docker ps -a
|
||||
|
||||
echo "=== Backend logs ==="
|
||||
docker logs fuzzforge-backend --tail 100
|
||||
|
||||
echo "=== Worker logs ==="
|
||||
for worker in python secrets android; do
|
||||
if docker ps -a --format "{{.Names}}" | grep -q "fuzzforge-worker-$worker"; then
|
||||
echo "=== Worker: $worker ==="
|
||||
docker logs fuzzforge-worker-$worker --tail 50
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
#############################################################################
|
||||
# Platform-Specific Tests (Android Worker)
|
||||
#############################################################################
|
||||
android-platform-tests:
|
||||
name: Android Worker Platform Tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs: platform-detection-tests
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
platform: linux/amd64
|
||||
arch: x86_64
|
||||
# ARM64 runner (uncomment when GitHub Actions ARM64 runners are available)
|
||||
# - os: ubuntu-24.04-arm
|
||||
# platform: linux/arm64
|
||||
# arch: aarch64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install FuzzForge CLI
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pyyaml
|
||||
# Install local monorepo dependencies first
|
||||
pip install -e ../sdk
|
||||
pip install -e ../ai
|
||||
# Then install CLI
|
||||
pip install -e .
|
||||
|
||||
- name: Verify platform detection
|
||||
run: |
|
||||
echo "Expected platform: ${{ matrix.platform }}"
|
||||
echo "Expected arch: ${{ matrix.arch }}"
|
||||
echo "Actual arch: $(uname -m)"
|
||||
|
||||
# Verify platform matches
|
||||
if [ "$(uname -m)" != "${{ matrix.arch }}" ]; then
|
||||
echo "❌ Platform mismatch!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Check Android worker Dockerfile selection
|
||||
run: |
|
||||
# Check which Dockerfile would be selected
|
||||
if [ "${{ matrix.platform }}" == "linux/amd64" ]; then
|
||||
expected_dockerfile="Dockerfile.amd64"
|
||||
else
|
||||
expected_dockerfile="Dockerfile.arm64"
|
||||
fi
|
||||
|
||||
echo "Expected Dockerfile: $expected_dockerfile"
|
||||
|
||||
# Verify the Dockerfile exists
|
||||
if [ ! -f "workers/android/$expected_dockerfile" ]; then
|
||||
echo "❌ Dockerfile not found: workers/android/$expected_dockerfile"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Dockerfile exists: $expected_dockerfile"
|
||||
|
||||
- name: Build Android worker for platform
|
||||
run: |
|
||||
echo "Building Android worker for platform: ${{ matrix.platform }}"
|
||||
docker compose build worker-android
|
||||
timeout-minutes: 15
|
||||
|
||||
- name: Copy environment template
|
||||
run: |
|
||||
mkdir -p volumes/env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
- name: Start FuzzForge services
|
||||
run: |
|
||||
docker compose up -d
|
||||
sleep 30
|
||||
|
||||
- name: Initialize test projects
|
||||
run: |
|
||||
echo "Initializing test projects..."
|
||||
mkdir -p test_projects/android_test/.fuzzforge
|
||||
cat > test_projects/android_test/.fuzzforge/config.yaml <<EOF
|
||||
project:
|
||||
name: android_test
|
||||
api_url: http://localhost:8000
|
||||
id: test-$(uuidgen | tr '[:upper:]' '[:lower:]' | tr -d '-' | head -c 16)
|
||||
EOF
|
||||
|
||||
- name: Run Android workflow test
|
||||
run: |
|
||||
python scripts/test_workflows.py \
|
||||
--workflow android_static_analysis \
|
||||
--platform ${{ matrix.platform }} \
|
||||
--skip-service-start
|
||||
timeout-minutes: 10
|
||||
|
||||
- name: Verify correct Dockerfile was used
|
||||
run: |
|
||||
# Check docker image labels or inspect to verify correct build
|
||||
docker inspect fuzzforge-worker-android | grep -i "dockerfile" || true
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Android worker logs ==="
|
||||
docker logs fuzzforge-worker-android --tail 100
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
#############################################################################
|
||||
# Full Workflow Tests (on schedule or manual trigger)
|
||||
#############################################################################
|
||||
full-workflow-tests:
|
||||
name: Full Workflow Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: platform-detection-tests
|
||||
# Only run full tests on schedule, manual trigger, or main branch
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install FuzzForge CLI
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pyyaml
|
||||
# Install local monorepo dependencies first
|
||||
pip install -e ../sdk
|
||||
pip install -e ../ai
|
||||
# Then install CLI
|
||||
pip install -e .
|
||||
|
||||
- name: Copy environment template
|
||||
run: |
|
||||
mkdir -p volumes/env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
- name: Start FuzzForge services
|
||||
run: |
|
||||
docker compose up -d
|
||||
sleep 30
|
||||
|
||||
- name: Initialize test projects
|
||||
run: |
|
||||
echo "Initializing test projects..."
|
||||
# Create minimal .fuzzforge directories for test projects
|
||||
for project in vulnerable_app android_test secret_detection_benchmark rust_test; do
|
||||
mkdir -p test_projects/$project/.fuzzforge
|
||||
cat > test_projects/$project/.fuzzforge/config.yaml <<EOF
|
||||
project:
|
||||
name: $project
|
||||
api_url: http://localhost:8000
|
||||
id: test-$(uuidgen | tr '[:upper:]' '[:lower:]' | tr -d '-' | head -c 16)
|
||||
EOF
|
||||
done
|
||||
cd ../rust_test && ff init || true
|
||||
|
||||
- name: Run full workflow tests
|
||||
run: |
|
||||
python scripts/test_workflows.py --suite full --skip-service-start
|
||||
timeout-minutes: 45
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Docker container status ==="
|
||||
docker ps -a
|
||||
|
||||
echo "=== All worker logs ==="
|
||||
for worker in python secrets rust android ossfuzz; do
|
||||
if docker ps -a --format "{{.Names}}" | grep -q "fuzzforge-worker-$worker"; then
|
||||
echo "=== Worker: $worker ==="
|
||||
docker logs fuzzforge-worker-$worker --tail 100
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
#############################################################################
|
||||
# Test Summary
|
||||
#############################################################################
|
||||
test-summary:
|
||||
name: Workflow Test Summary
|
||||
runs-on: ubuntu-latest
|
||||
needs: [platform-detection-tests, fast-workflow-tests, android-platform-tests]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Check test results
|
||||
run: |
|
||||
if [ "${{ needs.platform-detection-tests.result }}" != "success" ]; then
|
||||
echo "❌ Platform detection tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${{ needs.fast-workflow-tests.result }}" != "success" ]; then
|
||||
echo "❌ Fast workflow tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${{ needs.android-platform-tests.result }}" != "success" ]; then
|
||||
echo "❌ Android platform tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ All workflow integration tests passed!"
|
||||
8
.github/workflows/test.yml
vendored
8
.github/workflows/test.yml
vendored
@@ -1,5 +1,13 @@
|
||||
name: Tests
|
||||
|
||||
# This workflow covers:
|
||||
# - Worker validation (Dockerfile and metadata checks)
|
||||
# - Docker image builds (only for modified workers)
|
||||
# - Python linting (ruff, mypy)
|
||||
# - Backend unit tests
|
||||
#
|
||||
# For end-to-end workflow integration tests, see: .github/workflows/test-workflows.yml
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, dev, develop, feature/** ]
|
||||
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@@ -188,6 +188,10 @@ logs/
|
||||
# Docker volume configs (keep .env.example but ignore actual .env)
|
||||
volumes/env/.env
|
||||
|
||||
# Vendored proxy sources (kept locally for reference)
|
||||
ai/proxy/bifrost/
|
||||
ai/proxy/litellm/
|
||||
|
||||
# Test project databases and configurations
|
||||
test_projects/*/.fuzzforge/
|
||||
test_projects/*/findings.db*
|
||||
@@ -304,4 +308,4 @@ test_projects/*/.npmrc
|
||||
test_projects/*/.git-credentials
|
||||
test_projects/*/credentials.*
|
||||
test_projects/*/api_keys.*
|
||||
test_projects/*/ci-*.sh
|
||||
test_projects/*/ci-*.sh
|
||||
|
||||
@@ -137,6 +137,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fixed default parameters from metadata.yaml not being applied to workflows when no parameters provided
|
||||
- Fixed gitleaks workflow failing on uploaded directories without Git history
|
||||
- Fixed worker startup command suggestions (now uses `docker compose up -d` with service names)
|
||||
- Fixed missing `cognify_text` method in CogneeProjectIntegration
|
||||
|
||||
@@ -115,9 +115,11 @@ For containerized workflows, see the [Docker Installation Guide](https://docs.do
|
||||
For AI-powered workflows, configure your LLM API keys:
|
||||
|
||||
```bash
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
# Edit volumes/env/.env and add your API keys (OpenAI, Anthropic, Google, etc.)
|
||||
# Add your key to LITELLM_GEMINI_API_KEY
|
||||
```
|
||||
> Dont change the OPENAI_API_KEY default value, as it is used for the LLM proxy.
|
||||
|
||||
This is required for:
|
||||
- `llm_secret_detection` workflow
|
||||
@@ -150,7 +152,7 @@ git clone https://github.com/fuzzinglabs/fuzzforge_ai.git
|
||||
cd fuzzforge_ai
|
||||
|
||||
# 2. Copy the default LLM env config
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
# 3. Start FuzzForge with Temporal
|
||||
docker compose up -d
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# Default LiteLLM configuration
|
||||
LITELLM_MODEL=gemini/gemini-2.0-flash-001
|
||||
# LITELLM_PROVIDER=gemini
|
||||
|
||||
# API keys (uncomment and fill as needed)
|
||||
# GOOGLE_API_KEY=
|
||||
# OPENAI_API_KEY=
|
||||
# ANTHROPIC_API_KEY=
|
||||
# OPENROUTER_API_KEY=
|
||||
# MISTRAL_API_KEY=
|
||||
@@ -16,4 +16,9 @@ COPY . /app/agent_with_adk_format
|
||||
WORKDIR /app/agent_with_adk_format
|
||||
ENV PYTHONPATH=/app
|
||||
|
||||
# Copy and set up entrypoint
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
RUN chmod +x /docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
@@ -43,18 +43,34 @@ cd task_agent
|
||||
# cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` (or `.env.example`) and add your API keys. The agent must be restarted after changes so the values are picked up:
|
||||
Edit `.env` (or `.env.example`) and add your proxy + API keys. The agent must be restarted after changes so the values are picked up:
|
||||
```bash
|
||||
# Set default model
|
||||
LITELLM_MODEL=gemini/gemini-2.0-flash-001
|
||||
# Route every request through the proxy container (use http://localhost:10999 from the host)
|
||||
FF_LLM_PROXY_BASE_URL=http://llm-proxy:4000
|
||||
|
||||
# Add API keys for providers you want to use
|
||||
GOOGLE_API_KEY=your_google_api_key
|
||||
OPENAI_API_KEY=your_openai_api_key
|
||||
ANTHROPIC_API_KEY=your_anthropic_api_key
|
||||
OPENROUTER_API_KEY=your_openrouter_api_key
|
||||
# Default model + provider the agent boots with
|
||||
LITELLM_MODEL=openai/gpt-4o-mini
|
||||
LITELLM_PROVIDER=openai
|
||||
|
||||
# Virtual key issued by the proxy to the task agent (bootstrap replaces the placeholder)
|
||||
OPENAI_API_KEY=sk-proxy-default
|
||||
|
||||
# Upstream keys stay inside the proxy. Store real secrets under the LiteLLM
|
||||
# aliases and the bootstrapper mirrors them into .env.litellm for the proxy container.
|
||||
LITELLM_OPENAI_API_KEY=your_real_openai_api_key
|
||||
LITELLM_ANTHROPIC_API_KEY=your_real_anthropic_key
|
||||
LITELLM_GEMINI_API_KEY=your_real_gemini_key
|
||||
LITELLM_MISTRAL_API_KEY=your_real_mistral_key
|
||||
LITELLM_OPENROUTER_API_KEY=your_real_openrouter_key
|
||||
```
|
||||
|
||||
> When running the agent outside of Docker, swap `FF_LLM_PROXY_BASE_URL` to the host port (default `http://localhost:10999`).
|
||||
|
||||
The bootstrap container provisions LiteLLM, copies provider secrets into
|
||||
`volumes/env/.env.litellm`, and rewrites `volumes/env/.env` with the virtual key.
|
||||
Populate the `LITELLM_*_API_KEY` values before the first launch so the proxy can
|
||||
reach your upstream providers as soon as the bootstrap script runs.
|
||||
|
||||
### 2. Install Dependencies
|
||||
|
||||
```bash
|
||||
|
||||
31
ai/agents/task_agent/docker-entrypoint.sh
Normal file
31
ai/agents/task_agent/docker-entrypoint.sh
Normal file
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Wait for .env file to have keys (max 30 seconds)
|
||||
echo "[task-agent] Waiting for virtual keys to be provisioned..."
|
||||
for i in $(seq 1 30); do
|
||||
if [ -f /app/config/.env ]; then
|
||||
# Check if TASK_AGENT_API_KEY has a value (not empty)
|
||||
KEY=$(grep -E '^TASK_AGENT_API_KEY=' /app/config/.env | cut -d'=' -f2)
|
||||
if [ -n "$KEY" ] && [ "$KEY" != "" ]; then
|
||||
echo "[task-agent] Virtual keys found, loading environment..."
|
||||
# Export keys from .env file
|
||||
export TASK_AGENT_API_KEY="$KEY"
|
||||
export OPENAI_API_KEY=$(grep -E '^OPENAI_API_KEY=' /app/config/.env | cut -d'=' -f2)
|
||||
export FF_LLM_PROXY_BASE_URL=$(grep -E '^FF_LLM_PROXY_BASE_URL=' /app/config/.env | cut -d'=' -f2)
|
||||
echo "[task-agent] Loaded TASK_AGENT_API_KEY: ${TASK_AGENT_API_KEY:0:15}..."
|
||||
echo "[task-agent] Loaded FF_LLM_PROXY_BASE_URL: $FF_LLM_PROXY_BASE_URL"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
echo "[task-agent] Keys not ready yet, waiting... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ -z "$TASK_AGENT_API_KEY" ]; then
|
||||
echo "[task-agent] ERROR: Virtual keys were not provisioned within 30 seconds!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[task-agent] Starting uvicorn..."
|
||||
exec "$@"
|
||||
@@ -4,13 +4,28 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def _normalize_proxy_base_url(raw_value: str | None) -> str | None:
|
||||
if not raw_value:
|
||||
return None
|
||||
cleaned = raw_value.strip()
|
||||
if not cleaned:
|
||||
return None
|
||||
# Avoid double slashes in downstream requests
|
||||
return cleaned.rstrip("/")
|
||||
|
||||
AGENT_NAME = "litellm_agent"
|
||||
AGENT_DESCRIPTION = (
|
||||
"A LiteLLM-backed shell that exposes hot-swappable model and prompt controls."
|
||||
)
|
||||
|
||||
DEFAULT_MODEL = os.getenv("LITELLM_MODEL", "gemini-2.0-flash-001")
|
||||
DEFAULT_PROVIDER = os.getenv("LITELLM_PROVIDER")
|
||||
DEFAULT_MODEL = os.getenv("LITELLM_MODEL", "openai/gpt-4o-mini")
|
||||
DEFAULT_PROVIDER = os.getenv("LITELLM_PROVIDER") or None
|
||||
PROXY_BASE_URL = _normalize_proxy_base_url(
|
||||
os.getenv("FF_LLM_PROXY_BASE_URL")
|
||||
or os.getenv("LITELLM_API_BASE")
|
||||
or os.getenv("LITELLM_BASE_URL")
|
||||
)
|
||||
|
||||
STATE_PREFIX = "app:litellm_agent/"
|
||||
STATE_MODEL_KEY = f"{STATE_PREFIX}model"
|
||||
|
||||
@@ -3,11 +3,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from typing import Any, Mapping, MutableMapping, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import (
|
||||
DEFAULT_MODEL,
|
||||
DEFAULT_PROVIDER,
|
||||
PROXY_BASE_URL,
|
||||
STATE_MODEL_KEY,
|
||||
STATE_PROMPT_KEY,
|
||||
STATE_PROVIDER_KEY,
|
||||
@@ -66,11 +70,109 @@ class HotSwapState:
|
||||
"""Create a LiteLlm instance for the current state."""
|
||||
|
||||
from google.adk.models.lite_llm import LiteLlm # Lazy import to avoid cycle
|
||||
from google.adk.models.lite_llm import LiteLLMClient
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
kwargs = {"model": self.model}
|
||||
if self.provider:
|
||||
kwargs["custom_llm_provider"] = self.provider
|
||||
return LiteLlm(**kwargs)
|
||||
if PROXY_BASE_URL:
|
||||
provider = (self.provider or DEFAULT_PROVIDER or "").lower()
|
||||
if provider and provider != "openai":
|
||||
kwargs["api_base"] = f"{PROXY_BASE_URL.rstrip('/')}/{provider}"
|
||||
else:
|
||||
kwargs["api_base"] = PROXY_BASE_URL
|
||||
kwargs.setdefault("api_key", os.environ.get("TASK_AGENT_API_KEY") or os.environ.get("OPENAI_API_KEY"))
|
||||
|
||||
provider = (self.provider or DEFAULT_PROVIDER or "").lower()
|
||||
model_suffix = self.model.split("/", 1)[-1]
|
||||
use_responses = provider == "openai" and (
|
||||
model_suffix.startswith("gpt-5") or model_suffix.startswith("o1")
|
||||
)
|
||||
if use_responses:
|
||||
kwargs.setdefault("use_responses_api", True)
|
||||
|
||||
llm = LiteLlm(**kwargs)
|
||||
|
||||
if use_responses and PROXY_BASE_URL:
|
||||
|
||||
class _ResponsesAwareClient(LiteLLMClient):
|
||||
def __init__(self, base_client: LiteLLMClient, api_base: str, api_key: str):
|
||||
self._base_client = base_client
|
||||
self._api_base = api_base.rstrip("/")
|
||||
self._api_key = api_key
|
||||
|
||||
async def acompletion(self, model, messages, tools, **kwargs): # type: ignore[override]
|
||||
use_responses_api = kwargs.pop("use_responses_api", False)
|
||||
if not use_responses_api:
|
||||
return await self._base_client.acompletion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
resolved_model = model
|
||||
if "/" not in resolved_model:
|
||||
resolved_model = f"openai/{resolved_model}"
|
||||
|
||||
payload = {
|
||||
"model": resolved_model,
|
||||
"input": _messages_to_responses_input(messages),
|
||||
}
|
||||
|
||||
timeout = kwargs.get("timeout", 60)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self._api_base}/v1/responses",
|
||||
json=payload,
|
||||
headers=headers,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
text = exc.response.text
|
||||
raise RuntimeError(
|
||||
f"LiteLLM responses request failed: {text}"
|
||||
) from exc
|
||||
data = response.json()
|
||||
|
||||
text_output = _extract_output_text(data)
|
||||
usage = data.get("usage", {})
|
||||
|
||||
return ModelResponse(
|
||||
id=data.get("id"),
|
||||
model=model,
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(role="assistant", content=text_output),
|
||||
provider_specific_fields={"bifrost_response": data},
|
||||
)
|
||||
],
|
||||
usage=Usage(
|
||||
prompt_tokens=usage.get("input_tokens"),
|
||||
completion_tokens=usage.get("output_tokens"),
|
||||
reasoning_tokens=usage.get("output_tokens_details", {}).get(
|
||||
"reasoning_tokens"
|
||||
),
|
||||
total_tokens=usage.get("total_tokens"),
|
||||
),
|
||||
)
|
||||
|
||||
llm.llm_client = _ResponsesAwareClient(
|
||||
llm.llm_client,
|
||||
PROXY_BASE_URL,
|
||||
os.environ.get("TASK_AGENT_API_KEY") or os.environ.get("OPENAI_API_KEY", ""),
|
||||
)
|
||||
|
||||
return llm
|
||||
|
||||
@property
|
||||
def display_model(self) -> str:
|
||||
@@ -84,3 +186,69 @@ def apply_state_to_agent(invocation_context, state: HotSwapState) -> None:
|
||||
|
||||
agent = invocation_context.agent
|
||||
agent.model = state.instantiate_llm()
|
||||
|
||||
|
||||
def _messages_to_responses_input(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
inputs: list[dict[str, Any]] = []
|
||||
for message in messages:
|
||||
role = message.get("role", "user")
|
||||
content = message.get("content", "")
|
||||
text_segments: list[str] = []
|
||||
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
text = item.get("text") or item.get("content")
|
||||
if text:
|
||||
text_segments.append(str(text))
|
||||
elif isinstance(item, str):
|
||||
text_segments.append(item)
|
||||
elif isinstance(content, str):
|
||||
text_segments.append(content)
|
||||
|
||||
text = "\n".join(segment.strip() for segment in text_segments if segment)
|
||||
if not text:
|
||||
continue
|
||||
|
||||
entry_type = "input_text"
|
||||
if role == "assistant":
|
||||
entry_type = "output_text"
|
||||
|
||||
inputs.append(
|
||||
{
|
||||
"role": role,
|
||||
"content": [
|
||||
{
|
||||
"type": entry_type,
|
||||
"text": text,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
if not inputs:
|
||||
inputs.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
def _extract_output_text(response_json: dict[str, Any]) -> str:
|
||||
outputs = response_json.get("output", [])
|
||||
collected: list[str] = []
|
||||
for item in outputs:
|
||||
if isinstance(item, dict) and item.get("type") == "message":
|
||||
for part in item.get("content", []):
|
||||
if isinstance(part, dict) and part.get("type") == "output_text":
|
||||
text = part.get("text", "")
|
||||
if text:
|
||||
collected.append(str(text))
|
||||
return "\n\n".join(collected).strip()
|
||||
|
||||
5
ai/proxy/README.md
Normal file
5
ai/proxy/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
# LLM Proxy Integrations
|
||||
|
||||
This directory contains vendor source trees that were vendored only for reference when integrating LLM gateways. The actual FuzzForge deployment uses the official Docker images for each project.
|
||||
|
||||
See `docs/docs/how-to/llm-proxy.md` for up-to-date instructions on running the proxy services and issuing keys for the agents.
|
||||
@@ -1049,10 +1049,19 @@ class FuzzForgeExecutor:
|
||||
FunctionTool(get_task_list)
|
||||
])
|
||||
|
||||
|
||||
# Create the agent
|
||||
|
||||
# Create the agent with LiteLLM configuration
|
||||
llm_kwargs = {}
|
||||
api_key = os.getenv('OPENAI_API_KEY') or os.getenv('LLM_API_KEY')
|
||||
api_base = os.getenv('LLM_ENDPOINT') or os.getenv('LLM_API_BASE') or os.getenv('OPENAI_API_BASE')
|
||||
|
||||
if api_key:
|
||||
llm_kwargs['api_key'] = api_key
|
||||
if api_base:
|
||||
llm_kwargs['api_base'] = api_base
|
||||
|
||||
self.agent = LlmAgent(
|
||||
model=LiteLlm(model=self.model),
|
||||
model=LiteLlm(model=self.model, **llm_kwargs),
|
||||
name="fuzzforge_executor",
|
||||
description="Intelligent A2A orchestrator with memory",
|
||||
instruction=self._build_instruction(),
|
||||
|
||||
@@ -56,7 +56,7 @@ class CogneeService:
|
||||
# Configure LLM with API key BEFORE any other cognee operations
|
||||
provider = os.getenv("LLM_PROVIDER", "openai")
|
||||
model = os.getenv("LLM_MODEL") or os.getenv("LITELLM_MODEL", "gpt-4o-mini")
|
||||
api_key = os.getenv("LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
|
||||
api_key = os.getenv("COGNEE_API_KEY") or os.getenv("LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
|
||||
endpoint = os.getenv("LLM_ENDPOINT")
|
||||
api_version = os.getenv("LLM_API_VERSION")
|
||||
max_tokens = os.getenv("LLM_MAX_TOKENS")
|
||||
@@ -78,48 +78,62 @@ class CogneeService:
|
||||
os.environ.setdefault("OPENAI_API_KEY", api_key)
|
||||
if endpoint:
|
||||
os.environ["LLM_ENDPOINT"] = endpoint
|
||||
os.environ.setdefault("LLM_API_BASE", endpoint)
|
||||
os.environ.setdefault("OPENAI_API_BASE", endpoint)
|
||||
os.environ.setdefault("LITELLM_PROXY_API_BASE", endpoint)
|
||||
if api_key:
|
||||
os.environ.setdefault("LITELLM_PROXY_API_KEY", api_key)
|
||||
if api_version:
|
||||
os.environ["LLM_API_VERSION"] = api_version
|
||||
if max_tokens:
|
||||
os.environ["LLM_MAX_TOKENS"] = str(max_tokens)
|
||||
|
||||
# Configure Cognee's runtime using its configuration helpers when available
|
||||
embedding_model = os.getenv("LLM_EMBEDDING_MODEL")
|
||||
embedding_endpoint = os.getenv("LLM_EMBEDDING_ENDPOINT")
|
||||
if embedding_endpoint:
|
||||
os.environ.setdefault("LLM_EMBEDDING_API_BASE", embedding_endpoint)
|
||||
|
||||
if hasattr(cognee.config, "set_llm_provider"):
|
||||
cognee.config.set_llm_provider(provider)
|
||||
if hasattr(cognee.config, "set_llm_model"):
|
||||
cognee.config.set_llm_model(model)
|
||||
if api_key and hasattr(cognee.config, "set_llm_api_key"):
|
||||
cognee.config.set_llm_api_key(api_key)
|
||||
if endpoint and hasattr(cognee.config, "set_llm_endpoint"):
|
||||
cognee.config.set_llm_endpoint(endpoint)
|
||||
if hasattr(cognee.config, "set_llm_model"):
|
||||
cognee.config.set_llm_model(model)
|
||||
if api_key and hasattr(cognee.config, "set_llm_api_key"):
|
||||
cognee.config.set_llm_api_key(api_key)
|
||||
if endpoint and hasattr(cognee.config, "set_llm_endpoint"):
|
||||
cognee.config.set_llm_endpoint(endpoint)
|
||||
if embedding_model and hasattr(cognee.config, "set_llm_embedding_model"):
|
||||
cognee.config.set_llm_embedding_model(embedding_model)
|
||||
if embedding_endpoint and hasattr(cognee.config, "set_llm_embedding_endpoint"):
|
||||
cognee.config.set_llm_embedding_endpoint(embedding_endpoint)
|
||||
if api_version and hasattr(cognee.config, "set_llm_api_version"):
|
||||
cognee.config.set_llm_api_version(api_version)
|
||||
if max_tokens and hasattr(cognee.config, "set_llm_max_tokens"):
|
||||
cognee.config.set_llm_max_tokens(int(max_tokens))
|
||||
|
||||
|
||||
# Configure graph database
|
||||
cognee.config.set_graph_db_config({
|
||||
"graph_database_provider": self.cognee_config.get("graph_database_provider", "kuzu"),
|
||||
})
|
||||
|
||||
|
||||
# Set data directories
|
||||
data_dir = self.cognee_config.get("data_directory")
|
||||
system_dir = self.cognee_config.get("system_directory")
|
||||
|
||||
|
||||
if data_dir:
|
||||
logger.debug("Setting cognee data root", extra={"path": data_dir})
|
||||
cognee.config.data_root_directory(data_dir)
|
||||
if system_dir:
|
||||
logger.debug("Setting cognee system root", extra={"path": system_dir})
|
||||
cognee.config.system_root_directory(system_dir)
|
||||
|
||||
|
||||
# Setup multi-tenant user context
|
||||
await self._setup_user_context()
|
||||
|
||||
|
||||
self._initialized = True
|
||||
logger.info(f"Cognee initialized for project {self.project_context['project_name']} "
|
||||
f"with Kuzu at {system_dir}")
|
||||
|
||||
|
||||
except ImportError:
|
||||
logger.error("Cognee not installed. Install with: pip install cognee")
|
||||
raise
|
||||
|
||||
@@ -43,6 +43,42 @@ ALLOWED_CONTENT_TYPES = [
|
||||
router = APIRouter(prefix="/workflows", tags=["workflows"])
|
||||
|
||||
|
||||
def extract_defaults_from_json_schema(metadata: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract default parameter values from JSON Schema format.
|
||||
|
||||
Converts from:
|
||||
parameters:
|
||||
properties:
|
||||
param_name:
|
||||
default: value
|
||||
|
||||
To:
|
||||
{param_name: value}
|
||||
|
||||
Args:
|
||||
metadata: Workflow metadata dictionary
|
||||
|
||||
Returns:
|
||||
Dictionary of parameter defaults
|
||||
"""
|
||||
defaults = {}
|
||||
|
||||
# Check if there's a legacy default_parameters field
|
||||
if "default_parameters" in metadata:
|
||||
defaults.update(metadata["default_parameters"])
|
||||
|
||||
# Extract defaults from JSON Schema parameters
|
||||
parameters = metadata.get("parameters", {})
|
||||
properties = parameters.get("properties", {})
|
||||
|
||||
for param_name, param_spec in properties.items():
|
||||
if "default" in param_spec:
|
||||
defaults[param_name] = param_spec["default"]
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
def create_structured_error_response(
|
||||
error_type: str,
|
||||
message: str,
|
||||
@@ -164,7 +200,7 @@ async def get_workflow_metadata(
|
||||
author=metadata.get("author"),
|
||||
tags=metadata.get("tags", []),
|
||||
parameters=metadata.get("parameters", {}),
|
||||
default_parameters=metadata.get("default_parameters", {}),
|
||||
default_parameters=extract_defaults_from_json_schema(metadata),
|
||||
required_modules=metadata.get("required_modules", [])
|
||||
)
|
||||
|
||||
@@ -221,7 +257,7 @@ async def submit_workflow(
|
||||
# Merge default parameters with user parameters
|
||||
workflow_info = temporal_mgr.workflows[workflow_name]
|
||||
metadata = workflow_info.metadata or {}
|
||||
defaults = metadata.get("default_parameters", {})
|
||||
defaults = extract_defaults_from_json_schema(metadata)
|
||||
user_params = submission.parameters or {}
|
||||
workflow_params = {**defaults, **user_params}
|
||||
|
||||
@@ -450,7 +486,7 @@ async def upload_and_submit_workflow(
|
||||
# Merge default parameters with user parameters
|
||||
workflow_info = temporal_mgr.workflows.get(workflow_name)
|
||||
metadata = workflow_info.metadata or {}
|
||||
defaults = metadata.get("default_parameters", {})
|
||||
defaults = extract_defaults_from_json_schema(metadata)
|
||||
workflow_params = {**defaults, **workflow_params}
|
||||
|
||||
# Start workflow execution
|
||||
@@ -617,11 +653,8 @@ async def get_workflow_parameters(
|
||||
else:
|
||||
param_definitions = parameters_schema
|
||||
|
||||
# Add default values to the schema
|
||||
default_params = metadata.get("default_parameters", {})
|
||||
for param_name, param_schema in param_definitions.items():
|
||||
if isinstance(param_schema, dict) and param_name in default_params:
|
||||
param_schema["default"] = default_params[param_name]
|
||||
# Extract default values from JSON Schema
|
||||
default_params = extract_defaults_from_json_schema(metadata)
|
||||
|
||||
return {
|
||||
"workflow": workflow_name,
|
||||
|
||||
@@ -187,12 +187,28 @@ class TemporalManager:
|
||||
|
||||
# Add parameters in order based on metadata schema
|
||||
# This ensures parameters match the workflow signature order
|
||||
if workflow_params and 'parameters' in workflow_info.metadata:
|
||||
# Apply defaults from metadata.yaml if parameter not provided
|
||||
if 'parameters' in workflow_info.metadata:
|
||||
param_schema = workflow_info.metadata['parameters'].get('properties', {})
|
||||
logger.debug(f"Found {len(param_schema)} parameters in schema")
|
||||
# Iterate parameters in schema order and add values
|
||||
for param_name in param_schema.keys():
|
||||
param_value = workflow_params.get(param_name)
|
||||
param_spec = param_schema[param_name]
|
||||
|
||||
# Use provided param, or fall back to default from metadata
|
||||
if workflow_params and param_name in workflow_params:
|
||||
param_value = workflow_params[param_name]
|
||||
logger.debug(f"Using provided value for {param_name}: {param_value}")
|
||||
elif 'default' in param_spec:
|
||||
param_value = param_spec['default']
|
||||
logger.debug(f"Using default for {param_name}: {param_value}")
|
||||
else:
|
||||
param_value = None
|
||||
logger.debug(f"No value or default for {param_name}, using None")
|
||||
|
||||
workflow_args.append(param_value)
|
||||
else:
|
||||
logger.debug("No 'parameters' section found in workflow metadata")
|
||||
|
||||
# Determine task queue from workflow vertical
|
||||
vertical = workflow_info.metadata.get("vertical", "default")
|
||||
|
||||
@@ -107,7 +107,8 @@ class LLMSecretDetectorModule(BaseModule):
|
||||
)
|
||||
|
||||
agent_url = config.get("agent_url")
|
||||
if not agent_url or not isinstance(agent_url, str):
|
||||
# agent_url is optional - will have default from metadata.yaml
|
||||
if agent_url is not None and not isinstance(agent_url, str):
|
||||
raise ValueError("agent_url must be a valid URL string")
|
||||
|
||||
max_files = config.get("max_files", 20)
|
||||
@@ -131,14 +132,14 @@ class LLMSecretDetectorModule(BaseModule):
|
||||
|
||||
logger.info(f"Starting LLM secret detection in workspace: {workspace}")
|
||||
|
||||
# Extract configuration
|
||||
agent_url = config.get("agent_url", "http://fuzzforge-task-agent:8000/a2a/litellm_agent")
|
||||
llm_model = config.get("llm_model", "gpt-4o-mini")
|
||||
llm_provider = config.get("llm_provider", "openai")
|
||||
file_patterns = config.get("file_patterns", ["*.py", "*.js", "*.ts", "*.java", "*.go", "*.env", "*.yaml", "*.yml", "*.json", "*.xml", "*.ini", "*.sql", "*.properties", "*.sh", "*.bat", "*.config", "*.conf", "*.toml", "*id_rsa*", "*.txt"])
|
||||
max_files = config.get("max_files", 20)
|
||||
max_file_size = config.get("max_file_size", 30000)
|
||||
timeout = config.get("timeout", 30) # Reduced from 45s
|
||||
# Extract configuration (defaults come from metadata.yaml via API)
|
||||
agent_url = config["agent_url"]
|
||||
llm_model = config["llm_model"]
|
||||
llm_provider = config["llm_provider"]
|
||||
file_patterns = config["file_patterns"]
|
||||
max_files = config["max_files"]
|
||||
max_file_size = config["max_file_size"]
|
||||
timeout = config["timeout"]
|
||||
|
||||
# Find files to analyze
|
||||
# Skip files that are unlikely to contain secrets
|
||||
|
||||
@@ -19,26 +19,78 @@ parameters:
|
||||
agent_url:
|
||||
type: string
|
||||
description: "A2A agent endpoint URL"
|
||||
default: "http://fuzzforge-task-agent:8000/a2a/litellm_agent"
|
||||
llm_model:
|
||||
type: string
|
||||
description: "LLM model to use (e.g., gpt-4o-mini, claude-3-5-sonnet)"
|
||||
default: "gpt-5-mini"
|
||||
llm_provider:
|
||||
type: string
|
||||
description: "LLM provider (openai, anthropic, etc.)"
|
||||
default: "openai"
|
||||
file_patterns:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: "File patterns to analyze (e.g., ['*.py', '*.js'])"
|
||||
default:
|
||||
- "*.py"
|
||||
- "*.js"
|
||||
- "*.ts"
|
||||
- "*.jsx"
|
||||
- "*.tsx"
|
||||
- "*.java"
|
||||
- "*.go"
|
||||
- "*.rs"
|
||||
- "*.c"
|
||||
- "*.cpp"
|
||||
- "*.h"
|
||||
- "*.hpp"
|
||||
- "*.cs"
|
||||
- "*.php"
|
||||
- "*.rb"
|
||||
- "*.swift"
|
||||
- "*.kt"
|
||||
- "*.scala"
|
||||
- "*.env"
|
||||
- "*.yaml"
|
||||
- "*.yml"
|
||||
- "*.json"
|
||||
- "*.xml"
|
||||
- "*.ini"
|
||||
- "*.sql"
|
||||
- "*.properties"
|
||||
- "*.sh"
|
||||
- "*.bat"
|
||||
- "*.ps1"
|
||||
- "*.config"
|
||||
- "*.conf"
|
||||
- "*.toml"
|
||||
- "*id_rsa*"
|
||||
- "*id_dsa*"
|
||||
- "*id_ecdsa*"
|
||||
- "*id_ed25519*"
|
||||
- "*.pem"
|
||||
- "*.key"
|
||||
- "*.pub"
|
||||
- "*.txt"
|
||||
- "*.md"
|
||||
- "Dockerfile"
|
||||
- "docker-compose.yml"
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
description: "File patterns to analyze for security issues and secrets"
|
||||
max_files:
|
||||
type: integer
|
||||
description: "Maximum number of files to analyze"
|
||||
default: 10
|
||||
max_file_size:
|
||||
type: integer
|
||||
description: "Maximum file size in bytes"
|
||||
default: 100000
|
||||
timeout:
|
||||
type: integer
|
||||
description: "Timeout per file in seconds"
|
||||
default: 90
|
||||
|
||||
output_schema:
|
||||
type: object
|
||||
|
||||
@@ -30,5 +30,42 @@ parameters:
|
||||
type: integer
|
||||
default: 20
|
||||
|
||||
max_file_size:
|
||||
type: integer
|
||||
default: 30000
|
||||
description: "Maximum file size in bytes"
|
||||
|
||||
timeout:
|
||||
type: integer
|
||||
default: 30
|
||||
description: "Timeout per file in seconds"
|
||||
|
||||
file_patterns:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
default:
|
||||
- "*.py"
|
||||
- "*.js"
|
||||
- "*.ts"
|
||||
- "*.java"
|
||||
- "*.go"
|
||||
- "*.env"
|
||||
- "*.yaml"
|
||||
- "*.yml"
|
||||
- "*.json"
|
||||
- "*.xml"
|
||||
- "*.ini"
|
||||
- "*.sql"
|
||||
- "*.properties"
|
||||
- "*.sh"
|
||||
- "*.bat"
|
||||
- "*.config"
|
||||
- "*.conf"
|
||||
- "*.toml"
|
||||
- "*id_rsa*"
|
||||
- "*.txt"
|
||||
description: "File patterns to scan for secrets"
|
||||
|
||||
required_modules:
|
||||
- "llm_secret_detector"
|
||||
|
||||
@@ -17,6 +17,7 @@ class LlmSecretDetectionWorkflow:
|
||||
llm_model: Optional[str] = None,
|
||||
llm_provider: Optional[str] = None,
|
||||
max_files: Optional[int] = None,
|
||||
max_file_size: Optional[int] = None,
|
||||
timeout: Optional[int] = None,
|
||||
file_patterns: Optional[list] = None
|
||||
) -> Dict[str, Any]:
|
||||
@@ -67,6 +68,8 @@ class LlmSecretDetectionWorkflow:
|
||||
config["llm_provider"] = llm_provider
|
||||
if max_files:
|
||||
config["max_files"] = max_files
|
||||
if max_file_size:
|
||||
config["max_file_size"] = max_file_size
|
||||
if timeout:
|
||||
config["timeout"] = timeout
|
||||
if file_patterns:
|
||||
|
||||
@@ -12,3 +12,6 @@ Command modules for FuzzForge CLI.
|
||||
#
|
||||
# Additional attribution and requirements are provided in the NOTICE file.
|
||||
|
||||
from . import worker
|
||||
|
||||
__all__ = ["worker"]
|
||||
|
||||
@@ -187,19 +187,40 @@ def _ensure_env_file(fuzzforge_dir: Path, force: bool) -> None:
|
||||
|
||||
console.print("🧠 Configuring AI environment...")
|
||||
console.print(" • Default LLM provider: openai")
|
||||
console.print(" • Default LLM model: gpt-5-mini")
|
||||
console.print(" • Default LLM model: litellm_proxy/gpt-5-mini")
|
||||
console.print(" • To customise provider/model later, edit .fuzzforge/.env")
|
||||
|
||||
llm_provider = "openai"
|
||||
llm_model = "gpt-5-mini"
|
||||
llm_model = "litellm_proxy/gpt-5-mini"
|
||||
|
||||
# Check for global virtual keys from volumes/env/.env
|
||||
global_env_key = None
|
||||
for parent in fuzzforge_dir.parents:
|
||||
global_env = parent / "volumes" / "env" / ".env"
|
||||
if global_env.exists():
|
||||
try:
|
||||
for line in global_env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("OPENAI_API_KEY=") and "=" in line:
|
||||
key_value = line.split("=", 1)[1].strip()
|
||||
if key_value and not key_value.startswith("your-") and key_value.startswith("sk-"):
|
||||
global_env_key = key_value
|
||||
console.print(f" • Found virtual key in {global_env.relative_to(parent)}")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
|
||||
api_key = Prompt.ask(
|
||||
"OpenAI API key (leave blank to fill manually)",
|
||||
"OpenAI API key (leave blank to use global virtual key)" if global_env_key else "OpenAI API key (leave blank to fill manually)",
|
||||
default="",
|
||||
show_default=False,
|
||||
console=console,
|
||||
)
|
||||
|
||||
# Use global key if user didn't provide one
|
||||
if not api_key and global_env_key:
|
||||
api_key = global_env_key
|
||||
|
||||
session_db_path = fuzzforge_dir / "fuzzforge_sessions.db"
|
||||
session_db_rel = session_db_path.relative_to(fuzzforge_dir.parent)
|
||||
|
||||
@@ -210,14 +231,20 @@ def _ensure_env_file(fuzzforge_dir: Path, force: bool) -> None:
|
||||
f"LLM_PROVIDER={llm_provider}",
|
||||
f"LLM_MODEL={llm_model}",
|
||||
f"LITELLM_MODEL={llm_model}",
|
||||
"LLM_ENDPOINT=http://localhost:10999",
|
||||
"LLM_API_KEY=",
|
||||
"LLM_EMBEDDING_MODEL=litellm_proxy/text-embedding-3-large",
|
||||
"LLM_EMBEDDING_ENDPOINT=http://localhost:10999",
|
||||
f"OPENAI_API_KEY={api_key}",
|
||||
"FUZZFORGE_MCP_URL=http://localhost:8010/mcp",
|
||||
"",
|
||||
"# Cognee configuration mirrors the primary LLM by default",
|
||||
f"LLM_COGNEE_PROVIDER={llm_provider}",
|
||||
f"LLM_COGNEE_MODEL={llm_model}",
|
||||
f"LLM_COGNEE_API_KEY={api_key}",
|
||||
"LLM_COGNEE_ENDPOINT=",
|
||||
"LLM_COGNEE_ENDPOINT=http://localhost:10999",
|
||||
"LLM_COGNEE_API_KEY=",
|
||||
"LLM_COGNEE_EMBEDDING_MODEL=litellm_proxy/text-embedding-3-large",
|
||||
"LLM_COGNEE_EMBEDDING_ENDPOINT=http://localhost:10999",
|
||||
"COGNEE_MCP_URL=",
|
||||
"",
|
||||
"# Session persistence options: inmemory | sqlite",
|
||||
@@ -239,6 +266,8 @@ def _ensure_env_file(fuzzforge_dir: Path, force: bool) -> None:
|
||||
for line in env_lines:
|
||||
if line.startswith("OPENAI_API_KEY="):
|
||||
template_lines.append("OPENAI_API_KEY=")
|
||||
elif line.startswith("LLM_API_KEY="):
|
||||
template_lines.append("LLM_API_KEY=")
|
||||
elif line.startswith("LLM_COGNEE_API_KEY="):
|
||||
template_lines.append("LLM_COGNEE_API_KEY=")
|
||||
else:
|
||||
|
||||
225
cli/src/fuzzforge_cli/commands/worker.py
Normal file
225
cli/src/fuzzforge_cli/commands/worker.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Worker management commands for FuzzForge CLI.
|
||||
|
||||
Provides commands to start, stop, and list Temporal workers.
|
||||
"""
|
||||
# Copyright (c) 2025 FuzzingLabs
|
||||
#
|
||||
# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file
|
||||
# at the root of this repository for details.
|
||||
#
|
||||
# After the Change Date (four years from publication), this version of the
|
||||
# Licensed Work will be made available under the Apache License, Version 2.0.
|
||||
# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Additional attribution and requirements are provided in the NOTICE file.
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import typer
|
||||
from pathlib import Path
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from typing import Optional
|
||||
|
||||
from ..worker_manager import WorkerManager
|
||||
|
||||
console = Console()
|
||||
app = typer.Typer(
|
||||
name="worker",
|
||||
help="🔧 Manage Temporal workers",
|
||||
no_args_is_help=True,
|
||||
)
|
||||
|
||||
|
||||
@app.command("stop")
|
||||
def stop_workers(
|
||||
all: bool = typer.Option(
|
||||
False, "--all",
|
||||
help="Stop all workers (default behavior, flag for clarity)"
|
||||
)
|
||||
):
|
||||
"""
|
||||
🛑 Stop all running FuzzForge workers.
|
||||
|
||||
This command stops all worker containers using the proper Docker Compose
|
||||
profile flag to ensure workers are actually stopped (since they're in profiles).
|
||||
|
||||
Examples:
|
||||
$ ff worker stop
|
||||
$ ff worker stop --all
|
||||
"""
|
||||
try:
|
||||
worker_mgr = WorkerManager()
|
||||
success = worker_mgr.stop_all_workers()
|
||||
|
||||
if success:
|
||||
sys.exit(0)
|
||||
else:
|
||||
console.print("⚠️ Some workers may not have stopped properly", style="yellow")
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"❌ Error: {e}", style="red")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@app.command("list")
|
||||
def list_workers(
|
||||
all: bool = typer.Option(
|
||||
False, "--all", "-a",
|
||||
help="Show all workers (including stopped)"
|
||||
)
|
||||
):
|
||||
"""
|
||||
📋 List FuzzForge workers and their status.
|
||||
|
||||
By default, shows only running workers. Use --all to see all workers.
|
||||
|
||||
Examples:
|
||||
$ ff worker list
|
||||
$ ff worker list --all
|
||||
"""
|
||||
try:
|
||||
# Get list of running workers
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "--filter", "name=fuzzforge-worker-",
|
||||
"--format", "{{.Names}}\t{{.Status}}\t{{.RunningFor}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
|
||||
running_workers = []
|
||||
if result.stdout.strip():
|
||||
for line in result.stdout.strip().splitlines():
|
||||
parts = line.split('\t')
|
||||
if len(parts) >= 3:
|
||||
running_workers.append({
|
||||
"name": parts[0].replace("fuzzforge-worker-", ""),
|
||||
"status": "Running",
|
||||
"uptime": parts[2]
|
||||
})
|
||||
|
||||
# If --all, also get stopped workers
|
||||
stopped_workers = []
|
||||
if all:
|
||||
result_all = subprocess.run(
|
||||
["docker", "ps", "-a", "--filter", "name=fuzzforge-worker-",
|
||||
"--format", "{{.Names}}\t{{.Status}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
|
||||
all_worker_names = set()
|
||||
for line in result_all.stdout.strip().splitlines():
|
||||
parts = line.split('\t')
|
||||
if len(parts) >= 2:
|
||||
worker_name = parts[0].replace("fuzzforge-worker-", "")
|
||||
all_worker_names.add(worker_name)
|
||||
# If not running, it's stopped
|
||||
if not any(w["name"] == worker_name for w in running_workers):
|
||||
stopped_workers.append({
|
||||
"name": worker_name,
|
||||
"status": "Stopped",
|
||||
"uptime": "-"
|
||||
})
|
||||
|
||||
# Display results
|
||||
if not running_workers and not stopped_workers:
|
||||
console.print("ℹ️ No workers found", style="cyan")
|
||||
console.print("\n💡 Start a worker with: [cyan]docker compose up -d worker-<name>[/cyan]")
|
||||
console.print(" Or run a workflow, which auto-starts workers: [cyan]ff workflow run <workflow> <target>[/cyan]")
|
||||
return
|
||||
|
||||
# Create table
|
||||
table = Table(title="FuzzForge Workers", show_header=True, header_style="bold cyan")
|
||||
table.add_column("Worker", style="cyan", no_wrap=True)
|
||||
table.add_column("Status", style="green")
|
||||
table.add_column("Uptime", style="dim")
|
||||
|
||||
# Add running workers
|
||||
for worker in running_workers:
|
||||
table.add_row(
|
||||
worker["name"],
|
||||
f"[green]●[/green] {worker['status']}",
|
||||
worker["uptime"]
|
||||
)
|
||||
|
||||
# Add stopped workers if --all
|
||||
for worker in stopped_workers:
|
||||
table.add_row(
|
||||
worker["name"],
|
||||
f"[red]●[/red] {worker['status']}",
|
||||
worker["uptime"]
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Summary
|
||||
if running_workers:
|
||||
console.print(f"\n✅ {len(running_workers)} worker(s) running")
|
||||
if stopped_workers:
|
||||
console.print(f"⏹️ {len(stopped_workers)} worker(s) stopped")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"❌ Error listing workers: {e}", style="red")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@app.command("start")
|
||||
def start_worker(
|
||||
name: str = typer.Argument(
|
||||
...,
|
||||
help="Worker name (e.g., 'python', 'android', 'secrets')"
|
||||
),
|
||||
build: bool = typer.Option(
|
||||
False, "--build",
|
||||
help="Rebuild worker image before starting"
|
||||
)
|
||||
):
|
||||
"""
|
||||
🚀 Start a specific worker.
|
||||
|
||||
The worker name should be the vertical name (e.g., 'python', 'android', 'rust').
|
||||
|
||||
Examples:
|
||||
$ ff worker start python
|
||||
$ ff worker start android --build
|
||||
"""
|
||||
try:
|
||||
service_name = f"worker-{name}"
|
||||
|
||||
console.print(f"🚀 Starting worker: [cyan]{service_name}[/cyan]")
|
||||
|
||||
# Build docker compose command
|
||||
cmd = ["docker", "compose", "up", "-d"]
|
||||
if build:
|
||||
cmd.append("--build")
|
||||
cmd.append(service_name)
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
console.print(f"✅ Worker [cyan]{service_name}[/cyan] started successfully")
|
||||
else:
|
||||
console.print(f"❌ Failed to start worker: {result.stderr}", style="red")
|
||||
console.print(
|
||||
f"\n💡 Try manually: [yellow]docker compose up -d {service_name}[/yellow]",
|
||||
style="dim"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"❌ Error: {e}", style="red")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -28,6 +28,58 @@ try: # Optional dependency; fall back if not installed
|
||||
except ImportError: # pragma: no cover - optional dependency
|
||||
load_dotenv = None
|
||||
|
||||
|
||||
def _load_env_file_if_exists(path: Path, override: bool = False) -> bool:
|
||||
if not path.exists():
|
||||
return False
|
||||
# Always use manual parsing to handle empty values correctly
|
||||
try:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
continue
|
||||
key, value = stripped.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if override:
|
||||
# Only override if value is non-empty
|
||||
if value:
|
||||
os.environ[key] = value
|
||||
else:
|
||||
# Set if not already in environment and value is non-empty
|
||||
if key not in os.environ and value:
|
||||
os.environ[key] = value
|
||||
return True
|
||||
except Exception: # pragma: no cover - best effort fallback
|
||||
return False
|
||||
|
||||
|
||||
def _find_shared_env_file(project_dir: Path) -> Path | None:
|
||||
for directory in [project_dir] + list(project_dir.parents):
|
||||
candidate = directory / "volumes" / "env" / ".env"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def load_project_env(project_dir: Optional[Path] = None) -> Path | None:
|
||||
"""Load project-local env, falling back to shared volumes/env/.env."""
|
||||
|
||||
project_dir = Path(project_dir or Path.cwd())
|
||||
shared_env = _find_shared_env_file(project_dir)
|
||||
loaded_shared = False
|
||||
if shared_env:
|
||||
loaded_shared = _load_env_file_if_exists(shared_env, override=False)
|
||||
|
||||
project_env = project_dir / ".fuzzforge" / ".env"
|
||||
if _load_env_file_if_exists(project_env, override=True):
|
||||
return project_env
|
||||
|
||||
if loaded_shared:
|
||||
return shared_env
|
||||
|
||||
return None
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
@@ -312,23 +364,7 @@ class ProjectConfigManager:
|
||||
if not cognee.get("enabled", True):
|
||||
return
|
||||
|
||||
# Load project-specific environment overrides from .fuzzforge/.env if available
|
||||
env_file = self.project_dir / ".fuzzforge" / ".env"
|
||||
if env_file.exists():
|
||||
if load_dotenv:
|
||||
load_dotenv(env_file, override=False)
|
||||
else:
|
||||
try:
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if "=" not in stripped:
|
||||
continue
|
||||
key, value = stripped.split("=", 1)
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
except Exception: # pragma: no cover - best effort fallback
|
||||
pass
|
||||
load_project_env(self.project_dir)
|
||||
|
||||
backend_access = "true" if cognee.get("backend_access_control", True) else "false"
|
||||
os.environ["ENABLE_BACKEND_ACCESS_CONTROL"] = backend_access
|
||||
@@ -374,6 +410,17 @@ class ProjectConfigManager:
|
||||
"OPENAI_API_KEY",
|
||||
)
|
||||
endpoint = _env("LLM_COGNEE_ENDPOINT", "COGNEE_LLM_ENDPOINT", "LLM_ENDPOINT")
|
||||
embedding_model = _env(
|
||||
"LLM_COGNEE_EMBEDDING_MODEL",
|
||||
"COGNEE_LLM_EMBEDDING_MODEL",
|
||||
"LLM_EMBEDDING_MODEL",
|
||||
)
|
||||
embedding_endpoint = _env(
|
||||
"LLM_COGNEE_EMBEDDING_ENDPOINT",
|
||||
"COGNEE_LLM_EMBEDDING_ENDPOINT",
|
||||
"LLM_EMBEDDING_ENDPOINT",
|
||||
"LLM_ENDPOINT",
|
||||
)
|
||||
api_version = _env(
|
||||
"LLM_COGNEE_API_VERSION",
|
||||
"COGNEE_LLM_API_VERSION",
|
||||
@@ -398,6 +445,20 @@ class ProjectConfigManager:
|
||||
os.environ.setdefault("OPENAI_API_KEY", api_key)
|
||||
if endpoint:
|
||||
os.environ["LLM_ENDPOINT"] = endpoint
|
||||
os.environ.setdefault("LLM_API_BASE", endpoint)
|
||||
os.environ.setdefault("LLM_EMBEDDING_ENDPOINT", endpoint)
|
||||
os.environ.setdefault("LLM_EMBEDDING_API_BASE", endpoint)
|
||||
os.environ.setdefault("OPENAI_API_BASE", endpoint)
|
||||
# Set LiteLLM proxy environment variables for SDK usage
|
||||
os.environ.setdefault("LITELLM_PROXY_API_BASE", endpoint)
|
||||
if api_key:
|
||||
# Set LiteLLM proxy API key from the virtual key
|
||||
os.environ.setdefault("LITELLM_PROXY_API_KEY", api_key)
|
||||
if embedding_model:
|
||||
os.environ["LLM_EMBEDDING_MODEL"] = embedding_model
|
||||
if embedding_endpoint:
|
||||
os.environ["LLM_EMBEDDING_ENDPOINT"] = embedding_endpoint
|
||||
os.environ.setdefault("LLM_EMBEDDING_API_BASE", embedding_endpoint)
|
||||
if api_version:
|
||||
os.environ["LLM_API_VERSION"] = api_version
|
||||
if max_tokens:
|
||||
|
||||
@@ -19,6 +19,8 @@ from rich.traceback import install
|
||||
from typing import Optional, List
|
||||
import sys
|
||||
|
||||
from .config import load_project_env
|
||||
|
||||
from .commands import (
|
||||
workflows,
|
||||
workflow_exec,
|
||||
@@ -27,12 +29,16 @@ from .commands import (
|
||||
config as config_cmd,
|
||||
ai,
|
||||
ingest,
|
||||
worker,
|
||||
)
|
||||
from .fuzzy import enhanced_command_not_found_handler
|
||||
|
||||
# Install rich traceback handler
|
||||
install(show_locals=True)
|
||||
|
||||
# Ensure environment variables are available before command execution
|
||||
load_project_env()
|
||||
|
||||
# Create console for rich output
|
||||
console = Console()
|
||||
|
||||
@@ -329,6 +335,7 @@ app.add_typer(finding_app, name="finding", help="🔍 View and analyze findings"
|
||||
app.add_typer(monitor.app, name="monitor", help="📊 Real-time monitoring")
|
||||
app.add_typer(ai.app, name="ai", help="🤖 AI integration features")
|
||||
app.add_typer(ingest.app, name="ingest", help="🧠 Ingest knowledge into AI")
|
||||
app.add_typer(worker.app, name="worker", help="🔧 Manage Temporal workers")
|
||||
|
||||
# Help and utility commands
|
||||
@app.command()
|
||||
@@ -404,7 +411,7 @@ def main():
|
||||
'init', 'status', 'config', 'clean',
|
||||
'workflows', 'workflow',
|
||||
'findings', 'finding',
|
||||
'monitor', 'ai', 'ingest',
|
||||
'monitor', 'ai', 'ingest', 'worker',
|
||||
'version'
|
||||
]
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from typing import Optional, Dict, Any
|
||||
import requests
|
||||
import yaml
|
||||
from rich.console import Console
|
||||
from rich.status import Status
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
@@ -163,11 +164,25 @@ class WorkerManager:
|
||||
Platform string: "linux/amd64" or "linux/arm64"
|
||||
"""
|
||||
machine = platform.machine().lower()
|
||||
if machine in ["x86_64", "amd64"]:
|
||||
return "linux/amd64"
|
||||
elif machine in ["arm64", "aarch64"]:
|
||||
return "linux/arm64"
|
||||
return "unknown"
|
||||
system = platform.system().lower()
|
||||
|
||||
logger.debug(f"Platform detection: machine={machine}, system={system}")
|
||||
|
||||
# Normalize machine architecture
|
||||
if machine in ["x86_64", "amd64", "x64"]:
|
||||
detected = "linux/amd64"
|
||||
elif machine in ["arm64", "aarch64", "armv8", "arm64v8"]:
|
||||
detected = "linux/arm64"
|
||||
else:
|
||||
# Fallback to amd64 for unknown architectures
|
||||
logger.warning(
|
||||
f"Unknown architecture '{machine}' detected, falling back to linux/amd64. "
|
||||
f"Please report this issue if you're experiencing problems."
|
||||
)
|
||||
detected = "linux/amd64"
|
||||
|
||||
logger.info(f"Detected platform: {detected}")
|
||||
return detected
|
||||
|
||||
def _read_worker_metadata(self, vertical: str) -> dict:
|
||||
"""
|
||||
@@ -213,28 +228,39 @@ class WorkerManager:
|
||||
|
||||
platforms = metadata.get("platforms", {})
|
||||
|
||||
if not platforms:
|
||||
# Metadata exists but no platform definitions
|
||||
logger.debug(f"No platform definitions in metadata for {vertical}, using Dockerfile")
|
||||
return "Dockerfile"
|
||||
|
||||
# Try detected platform first
|
||||
if detected_platform in platforms:
|
||||
dockerfile = platforms[detected_platform].get("dockerfile", "Dockerfile")
|
||||
logger.debug(f"Selected {dockerfile} for {vertical} on {detected_platform}")
|
||||
logger.info(f"✓ Selected {dockerfile} for {vertical} on {detected_platform}")
|
||||
return dockerfile
|
||||
|
||||
# Fallback to default platform
|
||||
default_platform = metadata.get("default_platform", "linux/amd64")
|
||||
logger.warning(
|
||||
f"Platform {detected_platform} not found in metadata for {vertical}, "
|
||||
f"falling back to default: {default_platform}"
|
||||
)
|
||||
|
||||
if default_platform in platforms:
|
||||
dockerfile = platforms[default_platform].get("dockerfile", "Dockerfile.amd64")
|
||||
logger.debug(f"Using default platform {default_platform}: {dockerfile}")
|
||||
logger.info(f"Using default platform {default_platform}: {dockerfile}")
|
||||
return dockerfile
|
||||
|
||||
# Last resort
|
||||
# Last resort: just use Dockerfile
|
||||
logger.warning(f"No suitable Dockerfile found for {vertical}, using 'Dockerfile'")
|
||||
return "Dockerfile"
|
||||
|
||||
def _run_docker_compose(self, *args: str, env: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess:
|
||||
"""
|
||||
Run docker-compose command with optional environment variables.
|
||||
Run docker compose command with optional environment variables.
|
||||
|
||||
Args:
|
||||
*args: Arguments to pass to docker-compose
|
||||
*args: Arguments to pass to docker compose
|
||||
env: Optional environment variables to set
|
||||
|
||||
Returns:
|
||||
@@ -243,7 +269,7 @@ class WorkerManager:
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If command fails
|
||||
"""
|
||||
cmd = ["docker-compose", "-f", str(self.compose_file)] + list(args)
|
||||
cmd = ["docker", "compose", "-f", str(self.compose_file)] + list(args)
|
||||
logger.debug(f"Running: {' '.join(cmd)}")
|
||||
|
||||
# Merge with current environment
|
||||
@@ -342,9 +368,67 @@ class WorkerManager:
|
||||
console.print(f"❌ Unexpected error: {e}", style="red")
|
||||
return False
|
||||
|
||||
def _get_container_state(self, service_name: str) -> str:
|
||||
"""
|
||||
Get the current state of a container (running, created, restarting, etc.).
|
||||
|
||||
Args:
|
||||
service_name: Name of the Docker Compose service
|
||||
|
||||
Returns:
|
||||
Container state string (running, created, restarting, exited, etc.) or "unknown"
|
||||
"""
|
||||
try:
|
||||
container_name = self._service_to_container_name(service_name)
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Status}}", container_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
return "unknown"
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get container state: {e}")
|
||||
return "unknown"
|
||||
|
||||
def _get_health_status(self, container_name: str) -> str:
|
||||
"""
|
||||
Get container health status.
|
||||
|
||||
Args:
|
||||
container_name: Docker container name
|
||||
|
||||
Returns:
|
||||
Health status: "healthy", "unhealthy", "starting", "none", or "unknown"
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Health.Status}}", container_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return "unknown"
|
||||
|
||||
health_status = result.stdout.strip()
|
||||
|
||||
if health_status == "<no value>" or health_status == "":
|
||||
return "none" # No health check defined
|
||||
|
||||
return health_status # healthy, unhealthy, starting
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to check health: {e}")
|
||||
return "unknown"
|
||||
|
||||
def wait_for_worker_ready(self, service_name: str, timeout: Optional[int] = None) -> bool:
|
||||
"""
|
||||
Wait for a worker to be healthy and ready to process tasks.
|
||||
Shows live progress updates during startup.
|
||||
|
||||
Args:
|
||||
service_name: Name of the Docker Compose service
|
||||
@@ -352,56 +436,74 @@ class WorkerManager:
|
||||
|
||||
Returns:
|
||||
True if worker is ready, False if timeout reached
|
||||
|
||||
Raises:
|
||||
TimeoutError: If worker doesn't become ready within timeout
|
||||
"""
|
||||
timeout = timeout or self.startup_timeout
|
||||
start_time = time.time()
|
||||
container_name = self._service_to_container_name(service_name)
|
||||
last_status_msg = ""
|
||||
|
||||
console.print("⏳ Waiting for worker to be ready...")
|
||||
with Status("[bold cyan]Starting worker...", console=console, spinner="dots") as status:
|
||||
while time.time() - start_time < timeout:
|
||||
elapsed = int(time.time() - start_time)
|
||||
|
||||
# Get container state
|
||||
container_state = self._get_container_state(service_name)
|
||||
|
||||
# Get health status
|
||||
health_status = self._get_health_status(container_name)
|
||||
|
||||
# Build status message based on current state
|
||||
if container_state == "created":
|
||||
status_msg = f"[cyan]Worker starting... ({elapsed}s)[/cyan]"
|
||||
elif container_state == "restarting":
|
||||
status_msg = f"[yellow]Worker restarting... ({elapsed}s)[/yellow]"
|
||||
elif container_state == "running":
|
||||
if health_status == "starting":
|
||||
status_msg = f"[cyan]Worker running, health check starting... ({elapsed}s)[/cyan]"
|
||||
elif health_status == "unhealthy":
|
||||
status_msg = f"[yellow]Worker running, health check: unhealthy ({elapsed}s)[/yellow]"
|
||||
elif health_status == "healthy":
|
||||
status_msg = f"[green]Worker healthy! ({elapsed}s)[/green]"
|
||||
status.update(status_msg)
|
||||
console.print(f"✅ Worker ready: {service_name} (took {elapsed}s)")
|
||||
logger.info(f"Worker {service_name} is healthy (took {elapsed}s)")
|
||||
return True
|
||||
elif health_status == "none":
|
||||
# No health check defined, assume ready
|
||||
status_msg = f"[green]Worker running (no health check) ({elapsed}s)[/green]"
|
||||
status.update(status_msg)
|
||||
console.print(f"✅ Worker ready: {service_name} (took {elapsed}s)")
|
||||
logger.info(f"Worker {service_name} is running, no health check (took {elapsed}s)")
|
||||
return True
|
||||
else:
|
||||
status_msg = f"[cyan]Worker running ({elapsed}s)[/cyan]"
|
||||
elif not container_state or container_state == "exited":
|
||||
status_msg = f"[yellow]Waiting for container to start... ({elapsed}s)[/yellow]"
|
||||
else:
|
||||
status_msg = f"[cyan]Worker state: {container_state} ({elapsed}s)[/cyan]"
|
||||
|
||||
# Show helpful hints at certain intervals
|
||||
if elapsed == 10:
|
||||
status_msg += " [dim](pulling image if not cached)[/dim]"
|
||||
elif elapsed == 30:
|
||||
status_msg += " [dim](large images can take time)[/dim]"
|
||||
elif elapsed == 60:
|
||||
status_msg += " [dim](still working...)[/dim]"
|
||||
|
||||
# Update status if changed
|
||||
if status_msg != last_status_msg:
|
||||
status.update(status_msg)
|
||||
last_status_msg = status_msg
|
||||
logger.debug(f"Worker {service_name} - state: {container_state}, health: {health_status}")
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
# Check if container is running
|
||||
if not self.is_worker_running(service_name):
|
||||
logger.debug(f"Worker {service_name} not running yet")
|
||||
time.sleep(self.health_check_interval)
|
||||
continue
|
||||
|
||||
# Check container health status
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Health.Status}}", container_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
|
||||
health_status = result.stdout.strip()
|
||||
|
||||
# If no health check is defined, assume healthy after running
|
||||
if health_status == "<no value>" or health_status == "":
|
||||
logger.info(f"Worker {service_name} is running (no health check)")
|
||||
console.print(f"✅ Worker ready: {service_name}")
|
||||
return True
|
||||
|
||||
if health_status == "healthy":
|
||||
logger.info(f"Worker {service_name} is healthy")
|
||||
console.print(f"✅ Worker ready: {service_name}")
|
||||
return True
|
||||
|
||||
logger.debug(f"Worker {service_name} health: {health_status}")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to check health: {e}")
|
||||
|
||||
time.sleep(self.health_check_interval)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
logger.warning(f"Worker {service_name} did not become ready within {elapsed:.1f}s")
|
||||
console.print(f"⚠️ Worker startup timeout after {elapsed:.1f}s", style="yellow")
|
||||
return False
|
||||
# Timeout reached
|
||||
elapsed = int(time.time() - start_time)
|
||||
logger.warning(f"Worker {service_name} did not become ready within {elapsed}s")
|
||||
console.print(f"⚠️ Worker startup timeout after {elapsed}s", style="yellow")
|
||||
console.print(f" Last state: {container_state}, health: {health_status}", style="dim")
|
||||
return False
|
||||
|
||||
def stop_worker(self, service_name: str) -> bool:
|
||||
"""
|
||||
@@ -432,6 +534,75 @@ class WorkerManager:
|
||||
console.print(f"❌ Unexpected error: {e}", style="red")
|
||||
return False
|
||||
|
||||
def stop_all_workers(self) -> bool:
|
||||
"""
|
||||
Stop all running FuzzForge worker containers.
|
||||
|
||||
This uses `docker stop` to stop worker containers individually,
|
||||
avoiding the Docker Compose profile issue and preventing accidental
|
||||
shutdown of core services.
|
||||
|
||||
Returns:
|
||||
True if all workers stopped successfully, False otherwise
|
||||
"""
|
||||
try:
|
||||
console.print("🛑 Stopping all FuzzForge workers...")
|
||||
|
||||
# Get list of all running worker containers
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "--filter", "name=fuzzforge-worker-", "--format", "{{.Names}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
|
||||
running_workers = [name.strip() for name in result.stdout.splitlines() if name.strip()]
|
||||
|
||||
if not running_workers:
|
||||
console.print("✓ No workers running")
|
||||
return True
|
||||
|
||||
console.print(f"Found {len(running_workers)} running worker(s):")
|
||||
for worker in running_workers:
|
||||
console.print(f" - {worker}")
|
||||
|
||||
# Stop each worker container individually using docker stop
|
||||
# This is safer than docker compose down and won't affect core services
|
||||
failed_workers = []
|
||||
for worker in running_workers:
|
||||
try:
|
||||
logger.info(f"Stopping {worker}...")
|
||||
result = subprocess.run(
|
||||
["docker", "stop", worker],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30
|
||||
)
|
||||
console.print(f" ✓ Stopped {worker}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Failed to stop {worker}: {e.stderr}")
|
||||
failed_workers.append(worker)
|
||||
console.print(f" ✗ Failed to stop {worker}", style="red")
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(f"Timeout stopping {worker}")
|
||||
failed_workers.append(worker)
|
||||
console.print(f" ✗ Timeout stopping {worker}", style="red")
|
||||
|
||||
if failed_workers:
|
||||
console.print(f"\n⚠️ {len(failed_workers)} worker(s) failed to stop", style="yellow")
|
||||
console.print("💡 Try manually: docker stop " + " ".join(failed_workers), style="dim")
|
||||
return False
|
||||
|
||||
console.print("\n✅ All workers stopped")
|
||||
logger.info("All workers stopped successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error stopping workers: {e}")
|
||||
console.print(f"❌ Unexpected error: {e}", style="red")
|
||||
return False
|
||||
|
||||
def ensure_worker_running(
|
||||
self,
|
||||
worker_info: Dict[str, Any],
|
||||
|
||||
0
cli/tests/__init__.py
Normal file
0
cli/tests/__init__.py
Normal file
256
cli/tests/test_platform_detection.py
Normal file
256
cli/tests/test_platform_detection.py
Normal file
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
Unit tests for platform detection and Dockerfile selection in WorkerManager.
|
||||
|
||||
These tests verify that the WorkerManager correctly detects the platform
|
||||
and selects the appropriate Dockerfile for workers with platform-specific
|
||||
configurations (e.g., Android worker with separate AMD64 and ARM64 Dockerfiles).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch, mock_open
|
||||
import yaml
|
||||
|
||||
from fuzzforge_cli.worker_manager import WorkerManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def worker_manager(tmp_path):
|
||||
"""Create a WorkerManager instance for testing."""
|
||||
# Create a dummy docker-compose.yml for testing
|
||||
dummy_compose = tmp_path / "docker-compose.yml"
|
||||
dummy_compose.write_text("version: '3.8'\nservices: {}")
|
||||
return WorkerManager(compose_file=dummy_compose)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_android_metadata():
|
||||
"""Mock metadata.yaml content for Android worker."""
|
||||
return """
|
||||
name: android
|
||||
version: "1.0.0"
|
||||
description: "Android application security testing worker"
|
||||
default_platform: linux/amd64
|
||||
|
||||
platforms:
|
||||
linux/amd64:
|
||||
dockerfile: Dockerfile.amd64
|
||||
description: "Full Android toolchain with MobSF support"
|
||||
supported_tools:
|
||||
- jadx
|
||||
- opengrep
|
||||
- mobsf
|
||||
- frida
|
||||
- androguard
|
||||
|
||||
linux/arm64:
|
||||
dockerfile: Dockerfile.arm64
|
||||
description: "Android toolchain without MobSF (ARM64/Apple Silicon compatible)"
|
||||
supported_tools:
|
||||
- jadx
|
||||
- opengrep
|
||||
- frida
|
||||
- androguard
|
||||
disabled_tools:
|
||||
mobsf: "Incompatible with Rosetta 2 emulation"
|
||||
"""
|
||||
|
||||
|
||||
class TestPlatformDetection:
|
||||
"""Test platform detection logic."""
|
||||
|
||||
def test_detect_platform_linux_x86_64(self, worker_manager):
|
||||
"""Test platform detection on Linux x86_64."""
|
||||
with patch('platform.machine', return_value='x86_64'), \
|
||||
patch('platform.system', return_value='Linux'):
|
||||
platform = worker_manager._detect_platform()
|
||||
assert platform == 'linux/amd64'
|
||||
|
||||
def test_detect_platform_linux_aarch64(self, worker_manager):
|
||||
"""Test platform detection on Linux aarch64."""
|
||||
with patch('platform.machine', return_value='aarch64'), \
|
||||
patch('platform.system', return_value='Linux'):
|
||||
platform = worker_manager._detect_platform()
|
||||
assert platform == 'linux/arm64'
|
||||
|
||||
def test_detect_platform_darwin_arm64(self, worker_manager):
|
||||
"""Test platform detection on macOS Apple Silicon."""
|
||||
with patch('platform.machine', return_value='arm64'), \
|
||||
patch('platform.system', return_value='Darwin'):
|
||||
platform = worker_manager._detect_platform()
|
||||
assert platform == 'linux/arm64'
|
||||
|
||||
def test_detect_platform_darwin_x86_64(self, worker_manager):
|
||||
"""Test platform detection on macOS Intel."""
|
||||
with patch('platform.machine', return_value='x86_64'), \
|
||||
patch('platform.system', return_value='Darwin'):
|
||||
platform = worker_manager._detect_platform()
|
||||
assert platform == 'linux/amd64'
|
||||
|
||||
|
||||
class TestDockerfileSelection:
|
||||
"""Test Dockerfile selection logic."""
|
||||
|
||||
def test_select_dockerfile_with_metadata_amd64(self, worker_manager, mock_android_metadata):
|
||||
"""Test Dockerfile selection for AMD64 platform with metadata."""
|
||||
with patch('platform.machine', return_value='x86_64'), \
|
||||
patch('platform.system', return_value='Linux'), \
|
||||
patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=mock_android_metadata)):
|
||||
|
||||
dockerfile = worker_manager._select_dockerfile('android')
|
||||
assert 'Dockerfile.amd64' in str(dockerfile)
|
||||
|
||||
def test_select_dockerfile_with_metadata_arm64(self, worker_manager, mock_android_metadata):
|
||||
"""Test Dockerfile selection for ARM64 platform with metadata."""
|
||||
with patch('platform.machine', return_value='arm64'), \
|
||||
patch('platform.system', return_value='Darwin'), \
|
||||
patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=mock_android_metadata)):
|
||||
|
||||
dockerfile = worker_manager._select_dockerfile('android')
|
||||
assert 'Dockerfile.arm64' in str(dockerfile)
|
||||
|
||||
def test_select_dockerfile_without_metadata(self, worker_manager):
|
||||
"""Test Dockerfile selection for worker without metadata (uses default Dockerfile)."""
|
||||
with patch('pathlib.Path.exists', return_value=False):
|
||||
dockerfile = worker_manager._select_dockerfile('python')
|
||||
assert str(dockerfile).endswith('Dockerfile')
|
||||
assert 'Dockerfile.amd64' not in str(dockerfile)
|
||||
assert 'Dockerfile.arm64' not in str(dockerfile)
|
||||
|
||||
def test_select_dockerfile_fallback_to_default(self, worker_manager):
|
||||
"""Test Dockerfile selection falls back to default platform when current platform not found."""
|
||||
# Metadata with only amd64 support
|
||||
limited_metadata = """
|
||||
name: test-worker
|
||||
default_platform: linux/amd64
|
||||
platforms:
|
||||
linux/amd64:
|
||||
dockerfile: Dockerfile.amd64
|
||||
"""
|
||||
with patch('platform.machine', return_value='arm64'), \
|
||||
patch('platform.system', return_value='Darwin'), \
|
||||
patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=limited_metadata)):
|
||||
|
||||
# Should fall back to default_platform (amd64) since arm64 is not defined
|
||||
dockerfile = worker_manager._select_dockerfile('test-worker')
|
||||
assert 'Dockerfile.amd64' in str(dockerfile)
|
||||
|
||||
|
||||
class TestMetadataParsing:
|
||||
"""Test metadata.yaml parsing and handling."""
|
||||
|
||||
def test_parse_valid_metadata(self, worker_manager, mock_android_metadata):
|
||||
"""Test parsing valid metadata.yaml."""
|
||||
with patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=mock_android_metadata)):
|
||||
|
||||
metadata_path = Path("workers/android/metadata.yaml")
|
||||
with open(metadata_path, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
assert metadata['name'] == 'android'
|
||||
assert metadata['default_platform'] == 'linux/amd64'
|
||||
assert 'linux/amd64' in metadata['platforms']
|
||||
assert 'linux/arm64' in metadata['platforms']
|
||||
assert metadata['platforms']['linux/amd64']['dockerfile'] == 'Dockerfile.amd64'
|
||||
assert metadata['platforms']['linux/arm64']['dockerfile'] == 'Dockerfile.arm64'
|
||||
|
||||
def test_handle_missing_metadata(self, worker_manager):
|
||||
"""Test handling when metadata.yaml doesn't exist."""
|
||||
with patch('pathlib.Path.exists', return_value=False):
|
||||
# Should use default Dockerfile when metadata doesn't exist
|
||||
dockerfile = worker_manager._select_dockerfile('nonexistent-worker')
|
||||
assert str(dockerfile).endswith('Dockerfile')
|
||||
|
||||
def test_handle_malformed_metadata(self, worker_manager):
|
||||
"""Test handling malformed metadata.yaml."""
|
||||
malformed_yaml = "{ invalid: yaml: content:"
|
||||
|
||||
with patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=malformed_yaml)):
|
||||
|
||||
# Should fall back to default Dockerfile on YAML parse error
|
||||
dockerfile = worker_manager._select_dockerfile('broken-worker')
|
||||
assert str(dockerfile).endswith('Dockerfile')
|
||||
|
||||
|
||||
class TestWorkerStartWithPlatform:
|
||||
"""Test worker startup with platform-specific configuration."""
|
||||
|
||||
def test_start_android_worker_amd64(self, worker_manager, mock_android_metadata):
|
||||
"""Test starting Android worker on AMD64 platform."""
|
||||
with patch('platform.machine', return_value='x86_64'), \
|
||||
patch('platform.system', return_value='Linux'), \
|
||||
patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=mock_android_metadata)), \
|
||||
patch('subprocess.run') as mock_run:
|
||||
|
||||
mock_run.return_value = Mock(returncode=0)
|
||||
|
||||
# This would call _select_dockerfile internally
|
||||
dockerfile = worker_manager._select_dockerfile('android')
|
||||
assert 'Dockerfile.amd64' in str(dockerfile)
|
||||
|
||||
# Verify it would use MobSF-enabled image
|
||||
with open(Path("workers/android/metadata.yaml"), 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
tools = metadata['platforms']['linux/amd64']['supported_tools']
|
||||
assert 'mobsf' in tools
|
||||
|
||||
def test_start_android_worker_arm64(self, worker_manager, mock_android_metadata):
|
||||
"""Test starting Android worker on ARM64 platform."""
|
||||
with patch('platform.machine', return_value='arm64'), \
|
||||
patch('platform.system', return_value='Darwin'), \
|
||||
patch('pathlib.Path.exists', return_value=True), \
|
||||
patch('builtins.open', mock_open(read_data=mock_android_metadata)), \
|
||||
patch('subprocess.run') as mock_run:
|
||||
|
||||
mock_run.return_value = Mock(returncode=0)
|
||||
|
||||
# This would call _select_dockerfile internally
|
||||
dockerfile = worker_manager._select_dockerfile('android')
|
||||
assert 'Dockerfile.arm64' in str(dockerfile)
|
||||
|
||||
# Verify MobSF is disabled on ARM64
|
||||
with open(Path("workers/android/metadata.yaml"), 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
tools = metadata['platforms']['linux/arm64']['supported_tools']
|
||||
assert 'mobsf' not in tools
|
||||
assert 'mobsf' in metadata['platforms']['linux/arm64']['disabled_tools']
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestPlatformDetectionIntegration:
|
||||
"""Integration tests that verify actual platform detection."""
|
||||
|
||||
def test_current_platform_detection(self, worker_manager):
|
||||
"""Test that platform detection works on current platform."""
|
||||
platform = worker_manager._detect_platform()
|
||||
|
||||
# Should be one of the supported platforms
|
||||
assert platform in ['linux/amd64', 'linux/arm64']
|
||||
|
||||
# Should match the actual system
|
||||
import platform as sys_platform
|
||||
machine = sys_platform.machine()
|
||||
|
||||
if machine in ['x86_64', 'AMD64']:
|
||||
assert platform == 'linux/amd64'
|
||||
elif machine in ['aarch64', 'arm64']:
|
||||
assert platform == 'linux/arm64'
|
||||
|
||||
def test_android_metadata_exists(self):
|
||||
"""Test that Android worker metadata file exists."""
|
||||
metadata_path = Path(__file__).parent.parent.parent / "workers" / "android" / "metadata.yaml"
|
||||
assert metadata_path.exists(), "Android worker metadata.yaml should exist"
|
||||
|
||||
# Verify it's valid YAML
|
||||
with open(metadata_path, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
assert 'platforms' in metadata
|
||||
assert 'linux/amd64' in metadata['platforms']
|
||||
assert 'linux/arm64' in metadata['platforms']
|
||||
@@ -144,6 +144,103 @@ services:
|
||||
networks:
|
||||
- fuzzforge-network
|
||||
|
||||
# ============================================================================
|
||||
# LLM Proxy - LiteLLM Gateway
|
||||
# ============================================================================
|
||||
llm-proxy:
|
||||
image: ghcr.io/berriai/litellm:main-stable
|
||||
container_name: fuzzforge-llm-proxy
|
||||
depends_on:
|
||||
llm-proxy-db:
|
||||
condition: service_healthy
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
env_file:
|
||||
- ./volumes/env/.env
|
||||
environment:
|
||||
PORT: 4000
|
||||
DATABASE_URL: postgresql://litellm:litellm@llm-proxy-db:5432/litellm
|
||||
STORE_MODEL_IN_DB: "True"
|
||||
UI_USERNAME: ${UI_USERNAME:-fuzzforge}
|
||||
UI_PASSWORD: ${UI_PASSWORD:-fuzzforge123}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
||||
ANTHROPIC_API_KEY: ${LITELLM_ANTHROPIC_API_KEY:-}
|
||||
OPENAI_API_KEY: ${LITELLM_OPENAI_API_KEY:-}
|
||||
command:
|
||||
- "--config"
|
||||
- "/etc/litellm/proxy_config.yaml"
|
||||
ports:
|
||||
- "10999:4000" # Web UI + OpenAI-compatible API
|
||||
volumes:
|
||||
- litellm_proxy_data:/var/lib/litellm
|
||||
- ./volumes/litellm/proxy_config.yaml:/etc/litellm/proxy_config.yaml:ro
|
||||
networks:
|
||||
- fuzzforge-network
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector:latest
|
||||
container_name: fuzzforge-otel-collector
|
||||
command: ["--config=/etc/otel-collector/config.yaml"]
|
||||
volumes:
|
||||
- ./volumes/otel/collector-config.yaml:/etc/otel-collector/config.yaml:ro
|
||||
ports:
|
||||
- "4317:4317"
|
||||
- "4318:4318"
|
||||
networks:
|
||||
- fuzzforge-network
|
||||
restart: unless-stopped
|
||||
|
||||
llm-proxy-db:
|
||||
image: postgres:16
|
||||
container_name: fuzzforge-llm-proxy-db
|
||||
environment:
|
||||
POSTGRES_DB: litellm
|
||||
POSTGRES_USER: litellm
|
||||
POSTGRES_PASSWORD: litellm
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -d litellm -U litellm"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
volumes:
|
||||
- litellm_proxy_db:/var/lib/postgresql/data
|
||||
networks:
|
||||
- fuzzforge-network
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================================
|
||||
# LLM Proxy Bootstrap - Seed providers and virtual keys
|
||||
# ============================================================================
|
||||
llm-proxy-bootstrap:
|
||||
image: python:3.11-slim
|
||||
container_name: fuzzforge-llm-proxy-bootstrap
|
||||
depends_on:
|
||||
llm-proxy:
|
||||
condition: service_started
|
||||
env_file:
|
||||
- ./volumes/env/.env
|
||||
environment:
|
||||
PROXY_BASE_URL: http://llm-proxy:4000
|
||||
ENV_FILE_PATH: /bootstrap/env/.env
|
||||
UI_USERNAME: ${UI_USERNAME:-fuzzforge}
|
||||
UI_PASSWORD: ${UI_PASSWORD:-fuzzforge123}
|
||||
volumes:
|
||||
- ./docker/scripts/bootstrap_llm_proxy.py:/app/bootstrap.py:ro
|
||||
- ./volumes/env:/bootstrap/env
|
||||
- litellm_proxy_data:/bootstrap/data
|
||||
networks:
|
||||
- fuzzforge-network
|
||||
command: ["python", "/app/bootstrap.py"]
|
||||
restart: "no"
|
||||
|
||||
# ============================================================================
|
||||
# Vertical Worker: Rust/Native Security
|
||||
# ============================================================================
|
||||
@@ -458,10 +555,11 @@ services:
|
||||
context: ./ai/agents/task_agent
|
||||
dockerfile: Dockerfile
|
||||
container_name: fuzzforge-task-agent
|
||||
depends_on:
|
||||
llm-proxy-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
ports:
|
||||
- "10900:8000"
|
||||
env_file:
|
||||
- ./volumes/env/.env
|
||||
environment:
|
||||
- PORT=8000
|
||||
- PYTHONUNBUFFERED=1
|
||||
@@ -558,6 +656,10 @@ volumes:
|
||||
name: fuzzforge_worker_ossfuzz_cache
|
||||
worker_ossfuzz_build:
|
||||
name: fuzzforge_worker_ossfuzz_build
|
||||
litellm_proxy_data:
|
||||
name: fuzzforge_litellm_proxy_data
|
||||
litellm_proxy_db:
|
||||
name: fuzzforge_litellm_proxy_db
|
||||
# Add more worker caches as you add verticals:
|
||||
# worker_web_cache:
|
||||
# worker_ios_cache:
|
||||
@@ -591,6 +693,7 @@ networks:
|
||||
# 4. Web UIs:
|
||||
# - Temporal UI: http://localhost:8233
|
||||
# - MinIO Console: http://localhost:9001 (user: fuzzforge, pass: fuzzforge123)
|
||||
# - LiteLLM Proxy: http://localhost:10999
|
||||
#
|
||||
# 5. Resource Usage (Baseline):
|
||||
# - Temporal: ~500MB
|
||||
|
||||
636
docker/scripts/bootstrap_llm_proxy.py
Normal file
636
docker/scripts/bootstrap_llm_proxy.py
Normal file
@@ -0,0 +1,636 @@
|
||||
"""Bootstrap the LiteLLM proxy with provider secrets and default virtual keys.
|
||||
|
||||
The bootstrapper runs as a one-shot container during docker-compose startup.
|
||||
It performs the following actions:
|
||||
|
||||
1. Waits for the proxy health endpoint to respond.
|
||||
2. Collects upstream provider API keys from the shared .env file (plus any
|
||||
legacy copies) and mirrors them into a proxy-specific env file
|
||||
(volumes/env/.env.litellm) so only the proxy container can access them.
|
||||
3. Emits a default virtual key for the task agent by calling /key/generate,
|
||||
persisting the generated token back into volumes/env/.env so the agent can
|
||||
authenticate through the proxy instead of using raw provider secrets.
|
||||
4. Keeps the process idempotent: existing keys are reused and their allowed
|
||||
model list is refreshed instead of issuing duplicates on every run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
PROXY_BASE_URL = os.getenv("PROXY_BASE_URL", "http://llm-proxy:4000").rstrip("/")
|
||||
ENV_FILE_PATH = Path(os.getenv("ENV_FILE_PATH", "/bootstrap/env/.env"))
|
||||
LITELLM_ENV_FILE_PATH = Path(
|
||||
os.getenv("LITELLM_ENV_FILE_PATH", "/bootstrap/env/.env.litellm")
|
||||
)
|
||||
LEGACY_ENV_FILE_PATH = Path(
|
||||
os.getenv("LEGACY_ENV_FILE_PATH", "/bootstrap/env/.env.bifrost")
|
||||
)
|
||||
MAX_WAIT_SECONDS = int(os.getenv("LITELLM_PROXY_WAIT_SECONDS", "120"))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VirtualKeySpec:
|
||||
"""Configuration for a virtual key to be provisioned."""
|
||||
env_var: str
|
||||
alias: str
|
||||
user_id: str
|
||||
budget_env_var: str
|
||||
duration_env_var: str
|
||||
default_budget: float
|
||||
default_duration: str
|
||||
|
||||
|
||||
# Multiple virtual keys for different services
|
||||
VIRTUAL_KEYS: tuple[VirtualKeySpec, ...] = (
|
||||
VirtualKeySpec(
|
||||
env_var="OPENAI_API_KEY",
|
||||
alias="fuzzforge-cli",
|
||||
user_id="fuzzforge-cli",
|
||||
budget_env_var="CLI_BUDGET",
|
||||
duration_env_var="CLI_DURATION",
|
||||
default_budget=100.0,
|
||||
default_duration="30d",
|
||||
),
|
||||
VirtualKeySpec(
|
||||
env_var="TASK_AGENT_API_KEY",
|
||||
alias="fuzzforge-task-agent",
|
||||
user_id="fuzzforge-task-agent",
|
||||
budget_env_var="TASK_AGENT_BUDGET",
|
||||
duration_env_var="TASK_AGENT_DURATION",
|
||||
default_budget=25.0,
|
||||
default_duration="30d",
|
||||
),
|
||||
VirtualKeySpec(
|
||||
env_var="COGNEE_API_KEY",
|
||||
alias="fuzzforge-cognee",
|
||||
user_id="fuzzforge-cognee",
|
||||
budget_env_var="COGNEE_BUDGET",
|
||||
duration_env_var="COGNEE_DURATION",
|
||||
default_budget=50.0,
|
||||
default_duration="30d",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderSpec:
|
||||
name: str
|
||||
litellm_env_var: str
|
||||
alias_env_var: str
|
||||
source_env_vars: tuple[str, ...]
|
||||
|
||||
|
||||
# Support fresh LiteLLM variables while gracefully migrating legacy env
|
||||
# aliases on first boot.
|
||||
PROVIDERS: tuple[ProviderSpec, ...] = (
|
||||
ProviderSpec(
|
||||
"openai",
|
||||
"OPENAI_API_KEY",
|
||||
"LITELLM_OPENAI_API_KEY",
|
||||
("LITELLM_OPENAI_API_KEY", "BIFROST_OPENAI_KEY"),
|
||||
),
|
||||
ProviderSpec(
|
||||
"anthropic",
|
||||
"ANTHROPIC_API_KEY",
|
||||
"LITELLM_ANTHROPIC_API_KEY",
|
||||
("LITELLM_ANTHROPIC_API_KEY", "BIFROST_ANTHROPIC_KEY"),
|
||||
),
|
||||
ProviderSpec(
|
||||
"gemini",
|
||||
"GEMINI_API_KEY",
|
||||
"LITELLM_GEMINI_API_KEY",
|
||||
("LITELLM_GEMINI_API_KEY", "BIFROST_GEMINI_KEY"),
|
||||
),
|
||||
ProviderSpec(
|
||||
"mistral",
|
||||
"MISTRAL_API_KEY",
|
||||
"LITELLM_MISTRAL_API_KEY",
|
||||
("LITELLM_MISTRAL_API_KEY", "BIFROST_MISTRAL_KEY"),
|
||||
),
|
||||
ProviderSpec(
|
||||
"openrouter",
|
||||
"OPENROUTER_API_KEY",
|
||||
"LITELLM_OPENROUTER_API_KEY",
|
||||
("LITELLM_OPENROUTER_API_KEY", "BIFROST_OPENROUTER_KEY"),
|
||||
),
|
||||
)
|
||||
|
||||
PROVIDER_LOOKUP: dict[str, ProviderSpec] = {spec.name: spec for spec in PROVIDERS}
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
print(f"[litellm-bootstrap] {message}", flush=True)
|
||||
|
||||
|
||||
def read_lines(path: Path) -> list[str]:
|
||||
if not path.exists():
|
||||
return []
|
||||
return path.read_text().splitlines()
|
||||
|
||||
|
||||
def write_lines(path: Path, lines: Iterable[str]) -> None:
|
||||
material = "\n".join(lines)
|
||||
if material and not material.endswith("\n"):
|
||||
material += "\n"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(material)
|
||||
|
||||
|
||||
def read_env_file() -> list[str]:
|
||||
if not ENV_FILE_PATH.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Expected env file at {ENV_FILE_PATH}. Copy volumes/env/.env.template first."
|
||||
)
|
||||
return read_lines(ENV_FILE_PATH)
|
||||
|
||||
|
||||
def write_env_file(lines: Iterable[str]) -> None:
|
||||
write_lines(ENV_FILE_PATH, lines)
|
||||
|
||||
|
||||
def read_litellm_env_file() -> list[str]:
|
||||
return read_lines(LITELLM_ENV_FILE_PATH)
|
||||
|
||||
|
||||
def write_litellm_env_file(lines: Iterable[str]) -> None:
|
||||
write_lines(LITELLM_ENV_FILE_PATH, lines)
|
||||
|
||||
|
||||
def read_legacy_env_file() -> Mapping[str, str]:
|
||||
lines = read_lines(LEGACY_ENV_FILE_PATH)
|
||||
return parse_env_lines(lines)
|
||||
|
||||
|
||||
def set_env_value(lines: list[str], key: str, value: str) -> tuple[list[str], bool]:
|
||||
prefix = f"{key}="
|
||||
new_line = f"{prefix}{value}"
|
||||
for idx, line in enumerate(lines):
|
||||
stripped = line.lstrip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
if stripped.startswith(prefix):
|
||||
if stripped == new_line:
|
||||
return lines, False
|
||||
indent = line[: len(line) - len(stripped)]
|
||||
lines[idx] = f"{indent}{new_line}"
|
||||
return lines, True
|
||||
lines.append(new_line)
|
||||
return lines, True
|
||||
|
||||
|
||||
def parse_env_lines(lines: list[str]) -> dict[str, str]:
|
||||
mapping: dict[str, str] = {}
|
||||
for raw_line in lines:
|
||||
stripped = raw_line.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
continue
|
||||
key, value = stripped.split("=", 1)
|
||||
mapping[key] = value
|
||||
return mapping
|
||||
|
||||
|
||||
def wait_for_proxy() -> None:
|
||||
health_paths = ("/health/liveliness", "/health", "/")
|
||||
deadline = time.time() + MAX_WAIT_SECONDS
|
||||
attempt = 0
|
||||
while time.time() < deadline:
|
||||
attempt += 1
|
||||
for path in health_paths:
|
||||
url = f"{PROXY_BASE_URL}{path}"
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response: # noqa: S310
|
||||
if response.status < 400:
|
||||
log(f"Proxy responded on {path} (attempt {attempt})")
|
||||
return
|
||||
except urllib.error.URLError as exc:
|
||||
log(f"Proxy not ready yet ({path}): {exc}")
|
||||
time.sleep(3)
|
||||
raise TimeoutError(f"Timed out waiting for proxy at {PROXY_BASE_URL}")
|
||||
|
||||
|
||||
def request_json(
|
||||
path: str,
|
||||
*,
|
||||
method: str = "GET",
|
||||
payload: Mapping[str, object] | None = None,
|
||||
auth_token: str | None = None,
|
||||
) -> tuple[int, str]:
|
||||
url = f"{PROXY_BASE_URL}{path}"
|
||||
data = None
|
||||
headers = {"Accept": "application/json"}
|
||||
if auth_token:
|
||||
headers["Authorization"] = f"Bearer {auth_token}"
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request) as response: # noqa: S310
|
||||
body = response.read().decode("utf-8")
|
||||
return response.status, body
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8")
|
||||
return exc.code, body
|
||||
|
||||
|
||||
def get_master_key(env_map: Mapping[str, str]) -> str:
|
||||
candidate = os.getenv("LITELLM_MASTER_KEY") or env_map.get("LITELLM_MASTER_KEY")
|
||||
if not candidate:
|
||||
raise RuntimeError(
|
||||
"LITELLM_MASTER_KEY is not set. Add it to volumes/env/.env before starting Docker."
|
||||
)
|
||||
value = candidate.strip()
|
||||
if not value:
|
||||
raise RuntimeError(
|
||||
"LITELLM_MASTER_KEY is blank. Provide a non-empty value in the env file."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def gather_provider_keys(
|
||||
env_lines: list[str],
|
||||
env_map: dict[str, str],
|
||||
legacy_map: Mapping[str, str],
|
||||
) -> tuple[dict[str, str], list[str], bool]:
|
||||
updated_lines = list(env_lines)
|
||||
discovered: dict[str, str] = {}
|
||||
changed = False
|
||||
|
||||
for spec in PROVIDERS:
|
||||
value: str | None = None
|
||||
for source_var in spec.source_env_vars:
|
||||
candidate = env_map.get(source_var) or legacy_map.get(source_var) or os.getenv(
|
||||
source_var
|
||||
)
|
||||
if not candidate:
|
||||
continue
|
||||
stripped = candidate.strip()
|
||||
if stripped:
|
||||
value = stripped
|
||||
break
|
||||
if not value:
|
||||
continue
|
||||
|
||||
discovered[spec.litellm_env_var] = value
|
||||
updated_lines, alias_changed = set_env_value(
|
||||
updated_lines, spec.alias_env_var, value
|
||||
)
|
||||
if alias_changed:
|
||||
env_map[spec.alias_env_var] = value
|
||||
changed = True
|
||||
|
||||
return discovered, updated_lines, changed
|
||||
|
||||
|
||||
def ensure_litellm_env(provider_values: Mapping[str, str]) -> None:
|
||||
if not provider_values:
|
||||
log("No provider secrets discovered; skipping LiteLLM env update")
|
||||
return
|
||||
lines = read_litellm_env_file()
|
||||
updated_lines = list(lines)
|
||||
changed = False
|
||||
for env_var, value in provider_values.items():
|
||||
updated_lines, var_changed = set_env_value(updated_lines, env_var, value)
|
||||
if var_changed:
|
||||
changed = True
|
||||
if changed or not lines:
|
||||
write_litellm_env_file(updated_lines)
|
||||
log(f"Wrote provider secrets to {LITELLM_ENV_FILE_PATH}")
|
||||
|
||||
|
||||
def current_env_key(env_map: Mapping[str, str], env_var: str) -> str | None:
|
||||
candidate = os.getenv(env_var) or env_map.get(env_var)
|
||||
if not candidate:
|
||||
return None
|
||||
value = candidate.strip()
|
||||
if not value or value.startswith("sk-proxy-"):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def collect_default_models(env_map: Mapping[str, str]) -> list[str]:
|
||||
explicit = (
|
||||
os.getenv("LITELLM_DEFAULT_MODELS")
|
||||
or env_map.get("LITELLM_DEFAULT_MODELS")
|
||||
or ""
|
||||
)
|
||||
models: list[str] = []
|
||||
if explicit:
|
||||
models.extend(
|
||||
model.strip()
|
||||
for model in explicit.split(",")
|
||||
if model.strip()
|
||||
)
|
||||
if models:
|
||||
return sorted(dict.fromkeys(models))
|
||||
|
||||
configured_model = (
|
||||
os.getenv("LITELLM_MODEL") or env_map.get("LITELLM_MODEL") or ""
|
||||
).strip()
|
||||
configured_provider = (
|
||||
os.getenv("LITELLM_PROVIDER") or env_map.get("LITELLM_PROVIDER") or ""
|
||||
).strip()
|
||||
|
||||
if configured_model:
|
||||
if "/" in configured_model:
|
||||
models.append(configured_model)
|
||||
elif configured_provider:
|
||||
models.append(f"{configured_provider}/{configured_model}")
|
||||
else:
|
||||
log(
|
||||
"LITELLM_MODEL is set without a provider; configure LITELLM_PROVIDER or "
|
||||
"use the provider/model format (e.g. openai/gpt-4o-mini)."
|
||||
)
|
||||
elif configured_provider:
|
||||
log(
|
||||
"LITELLM_PROVIDER configured without a default model. Bootstrap will issue an "
|
||||
"unrestricted virtual key allowing any proxy-registered model."
|
||||
)
|
||||
|
||||
return sorted(dict.fromkeys(models))
|
||||
|
||||
|
||||
def fetch_existing_key_record(master_key: str, key_value: str) -> Mapping[str, object] | None:
|
||||
encoded = urllib.parse.quote_plus(key_value)
|
||||
status, body = request_json(f"/key/info?key={encoded}", auth_token=master_key)
|
||||
if status != 200:
|
||||
log(f"Key lookup failed ({status}); treating OPENAI_API_KEY as new")
|
||||
return None
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
log("Key info response was not valid JSON; ignoring")
|
||||
return None
|
||||
if isinstance(data, Mapping) and data.get("key"):
|
||||
return data
|
||||
return None
|
||||
|
||||
|
||||
def fetch_key_by_alias(master_key: str, alias: str) -> str | None:
|
||||
"""Fetch existing key value by alias from LiteLLM proxy."""
|
||||
status, body = request_json("/key/info", auth_token=master_key)
|
||||
if status != 200:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if isinstance(data, dict) and "keys" in data:
|
||||
for key_info in data.get("keys", []):
|
||||
if isinstance(key_info, dict) and key_info.get("key_alias") == alias:
|
||||
return str(key_info.get("key", "")).strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def generate_virtual_key(
|
||||
master_key: str,
|
||||
models: list[str],
|
||||
spec: VirtualKeySpec,
|
||||
env_map: Mapping[str, str],
|
||||
) -> str:
|
||||
budget_str = os.getenv(spec.budget_env_var) or env_map.get(spec.budget_env_var) or str(spec.default_budget)
|
||||
try:
|
||||
budget = float(budget_str)
|
||||
except ValueError:
|
||||
budget = spec.default_budget
|
||||
|
||||
duration = os.getenv(spec.duration_env_var) or env_map.get(spec.duration_env_var) or spec.default_duration
|
||||
|
||||
payload: dict[str, object] = {
|
||||
"key_alias": spec.alias,
|
||||
"user_id": spec.user_id,
|
||||
"duration": duration,
|
||||
"max_budget": budget,
|
||||
"metadata": {
|
||||
"provisioned_by": "bootstrap",
|
||||
"service": spec.alias,
|
||||
"default_models": models,
|
||||
},
|
||||
"key_type": "llm_api",
|
||||
}
|
||||
if models:
|
||||
payload["models"] = models
|
||||
status, body = request_json(
|
||||
"/key/generate", method="POST", payload=payload, auth_token=master_key
|
||||
)
|
||||
if status == 400 and "already exists" in body.lower():
|
||||
# Key alias already exists but .env is out of sync (e.g., after docker prune)
|
||||
# Delete the old key and regenerate
|
||||
log(f"Key alias '{spec.alias}' already exists in database but not in .env; deleting and regenerating")
|
||||
# Try to delete by alias using POST /key/delete with key_aliases array
|
||||
delete_payload = {"key_aliases": [spec.alias]}
|
||||
delete_status, delete_body = request_json(
|
||||
"/key/delete", method="POST", payload=delete_payload, auth_token=master_key
|
||||
)
|
||||
if delete_status not in {200, 201}:
|
||||
log(f"Warning: Could not delete existing key alias {spec.alias} ({delete_status}): {delete_body}")
|
||||
# Continue anyway and try to generate
|
||||
else:
|
||||
log(f"Deleted existing key alias {spec.alias}")
|
||||
|
||||
# Retry generation
|
||||
status, body = request_json(
|
||||
"/key/generate", method="POST", payload=payload, auth_token=master_key
|
||||
)
|
||||
if status not in {200, 201}:
|
||||
raise RuntimeError(f"Failed to generate virtual key for {spec.alias} ({status}): {body}")
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"Virtual key response for {spec.alias} was not valid JSON") from exc
|
||||
if isinstance(data, Mapping):
|
||||
key_value = str(data.get("key") or data.get("token") or "").strip()
|
||||
if key_value:
|
||||
log(f"Generated new LiteLLM virtual key for {spec.alias} (budget: ${budget}, duration: {duration})")
|
||||
return key_value
|
||||
raise RuntimeError(f"Virtual key response for {spec.alias} did not include a key field")
|
||||
|
||||
|
||||
def update_virtual_key(
|
||||
master_key: str,
|
||||
key_value: str,
|
||||
models: list[str],
|
||||
spec: VirtualKeySpec,
|
||||
) -> None:
|
||||
if not models:
|
||||
return
|
||||
payload: dict[str, object] = {
|
||||
"key": key_value,
|
||||
"models": models,
|
||||
}
|
||||
status, body = request_json(
|
||||
"/key/update", method="POST", payload=payload, auth_token=master_key
|
||||
)
|
||||
if status != 200:
|
||||
log(f"Virtual key update for {spec.alias} skipped ({status}): {body}")
|
||||
else:
|
||||
log(f"Refreshed allowed models for {spec.alias}")
|
||||
|
||||
|
||||
def persist_key_to_env(new_key: str, env_var: str) -> None:
|
||||
lines = read_env_file()
|
||||
updated_lines, changed = set_env_value(lines, env_var, new_key)
|
||||
# Always update the environment variable, even if file wasn't changed
|
||||
os.environ[env_var] = new_key
|
||||
if changed:
|
||||
write_env_file(updated_lines)
|
||||
log(f"Persisted {env_var} to {ENV_FILE_PATH}")
|
||||
else:
|
||||
log(f"{env_var} already up-to-date in env file")
|
||||
|
||||
|
||||
def ensure_virtual_key(
|
||||
master_key: str,
|
||||
models: list[str],
|
||||
env_map: Mapping[str, str],
|
||||
spec: VirtualKeySpec,
|
||||
) -> str:
|
||||
allowed_models: list[str] = []
|
||||
sync_flag = os.getenv("LITELLM_SYNC_VIRTUAL_KEY_MODELS", "").strip().lower()
|
||||
if models and (sync_flag in {"1", "true", "yes", "on"} or models == ["*"]):
|
||||
allowed_models = models
|
||||
existing_key = current_env_key(env_map, spec.env_var)
|
||||
if existing_key:
|
||||
record = fetch_existing_key_record(master_key, existing_key)
|
||||
if record:
|
||||
log(f"Reusing existing LiteLLM virtual key for {spec.alias}")
|
||||
if allowed_models:
|
||||
update_virtual_key(master_key, existing_key, allowed_models, spec)
|
||||
return existing_key
|
||||
log(f"Existing {spec.env_var} not registered with proxy; generating new key")
|
||||
|
||||
new_key = generate_virtual_key(master_key, models, spec, env_map)
|
||||
if allowed_models:
|
||||
update_virtual_key(master_key, new_key, allowed_models, spec)
|
||||
return new_key
|
||||
|
||||
|
||||
def _split_model_identifier(model: str) -> tuple[str | None, str]:
|
||||
if "/" in model:
|
||||
provider, short_name = model.split("/", 1)
|
||||
return provider.lower().strip() or None, short_name.strip()
|
||||
return None, model.strip()
|
||||
|
||||
|
||||
def ensure_models_registered(
|
||||
master_key: str,
|
||||
models: list[str],
|
||||
env_map: Mapping[str, str],
|
||||
) -> None:
|
||||
if not models:
|
||||
return
|
||||
for model in models:
|
||||
provider, short_name = _split_model_identifier(model)
|
||||
if not provider or not short_name:
|
||||
log(f"Skipping model '{model}' (no provider segment)")
|
||||
continue
|
||||
spec = PROVIDER_LOOKUP.get(provider)
|
||||
if not spec:
|
||||
log(f"No provider spec registered for '{provider}'; skipping model '{model}'")
|
||||
continue
|
||||
provider_secret = (
|
||||
env_map.get(spec.alias_env_var)
|
||||
or env_map.get(spec.litellm_env_var)
|
||||
or os.getenv(spec.alias_env_var)
|
||||
or os.getenv(spec.litellm_env_var)
|
||||
)
|
||||
if not provider_secret:
|
||||
log(
|
||||
f"Provider secret for '{provider}' not found; skipping model registration"
|
||||
)
|
||||
continue
|
||||
|
||||
api_key_reference = f"os.environ/{spec.alias_env_var}"
|
||||
payload: dict[str, object] = {
|
||||
"model_name": model,
|
||||
"litellm_params": {
|
||||
"model": short_name,
|
||||
"custom_llm_provider": provider,
|
||||
"api_key": api_key_reference,
|
||||
},
|
||||
"model_info": {
|
||||
"provider": provider,
|
||||
"description": "Auto-registered during bootstrap",
|
||||
},
|
||||
}
|
||||
|
||||
status, body = request_json(
|
||||
"/model/new", method="POST", payload=payload, auth_token=master_key
|
||||
)
|
||||
if status in {200, 201}:
|
||||
log(f"Registered LiteLLM model '{model}'")
|
||||
continue
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
data = body
|
||||
error_message = (
|
||||
data.get("error") if isinstance(data, Mapping) else str(data)
|
||||
)
|
||||
if status == 409 or (
|
||||
isinstance(error_message, str)
|
||||
and "already" in error_message.lower()
|
||||
):
|
||||
log(f"Model '{model}' already present; skipping")
|
||||
continue
|
||||
log(f"Failed to register model '{model}' ({status}): {error_message}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
log("Bootstrapping LiteLLM proxy")
|
||||
try:
|
||||
wait_for_proxy()
|
||||
env_lines = read_env_file()
|
||||
env_map = parse_env_lines(env_lines)
|
||||
legacy_map = read_legacy_env_file()
|
||||
master_key = get_master_key(env_map)
|
||||
|
||||
provider_values, updated_env_lines, env_changed = gather_provider_keys(
|
||||
env_lines, env_map, legacy_map
|
||||
)
|
||||
if env_changed:
|
||||
write_env_file(updated_env_lines)
|
||||
env_map = parse_env_lines(updated_env_lines)
|
||||
log("Updated LiteLLM provider aliases in shared env file")
|
||||
|
||||
ensure_litellm_env(provider_values)
|
||||
|
||||
models = collect_default_models(env_map)
|
||||
if models:
|
||||
log("Default models for virtual keys: %s" % ", ".join(models))
|
||||
models_for_key = models
|
||||
else:
|
||||
log(
|
||||
"No default models configured; provisioning virtual keys without model "
|
||||
"restrictions (model-agnostic)."
|
||||
)
|
||||
models_for_key = ["*"]
|
||||
|
||||
# Generate virtual keys for each service
|
||||
for spec in VIRTUAL_KEYS:
|
||||
virtual_key = ensure_virtual_key(master_key, models_for_key, env_map, spec)
|
||||
persist_key_to_env(virtual_key, spec.env_var)
|
||||
|
||||
# Register models if any were specified
|
||||
if models:
|
||||
ensure_models_registered(master_key, models, env_map)
|
||||
|
||||
log("Bootstrap complete")
|
||||
return 0
|
||||
except Exception as exc: # pragma: no cover - startup failure reported to logs
|
||||
log(f"Bootstrap failed: {exc}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -225,7 +225,7 @@ docker compose up -d # All workers start
|
||||
Set up AI workflows with API keys:
|
||||
|
||||
```bash
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
# Edit .env and add your API keys (OpenAI, Anthropic, etc.)
|
||||
```
|
||||
|
||||
|
||||
558
docs/docs/development/testing.md
Normal file
558
docs/docs/development/testing.md
Normal file
@@ -0,0 +1,558 @@
|
||||
# Testing Guide
|
||||
|
||||
This guide explains FuzzForge's testing infrastructure, including unit tests, workflow integration tests, and platform-specific testing for multi-architecture support.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
FuzzForge has multiple layers of testing:
|
||||
|
||||
1. **Unit Tests** - Backend and CLI unit tests
|
||||
2. **Worker Validation** - Docker image and metadata validation
|
||||
3. **Platform Detection Tests** - Verify correct Dockerfile selection across platforms
|
||||
4. **Workflow Integration Tests** - End-to-end workflow execution validation
|
||||
5. **Multi-Platform Tests** - Verify platform-specific Docker images (AMD64 vs ARM64)
|
||||
|
||||
---
|
||||
|
||||
## Test Organization
|
||||
|
||||
```
|
||||
.github/
|
||||
├── workflows/
|
||||
│ ├── test.yml # Unit tests, linting, worker builds
|
||||
│ └── test-workflows.yml # Workflow integration tests
|
||||
├── test-matrix.yaml # Workflow test configuration
|
||||
└── scripts/
|
||||
└── validate-workers.sh # Worker validation script
|
||||
|
||||
cli/
|
||||
└── tests/
|
||||
└── test_platform_detection.py # Platform detection unit tests
|
||||
|
||||
backend/
|
||||
└── tests/
|
||||
├── unit/ # Backend unit tests
|
||||
└── integration/ # Backend integration tests (commented out)
|
||||
|
||||
scripts/
|
||||
└── test_workflows.py # Workflow execution test script
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Running Tests Locally
|
||||
|
||||
### Prerequisites
|
||||
|
||||
```bash
|
||||
# Start FuzzForge services
|
||||
docker compose up -d
|
||||
|
||||
# Install CLI in development mode
|
||||
cd cli
|
||||
pip install -e ".[dev]"
|
||||
pip install pytest pytest-cov pyyaml
|
||||
```
|
||||
|
||||
### Unit Tests
|
||||
|
||||
#### Backend Unit Tests
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
pytest tests/unit/ -v \
|
||||
--cov=toolbox/modules \
|
||||
--cov=src \
|
||||
--cov-report=html
|
||||
```
|
||||
|
||||
#### CLI Platform Detection Tests
|
||||
|
||||
```bash
|
||||
cd cli
|
||||
pytest tests/test_platform_detection.py -v
|
||||
```
|
||||
|
||||
### Workflow Integration Tests
|
||||
|
||||
#### Run Fast Test Suite
|
||||
|
||||
Tests a subset of fast-running workflows:
|
||||
|
||||
```bash
|
||||
python scripts/test_workflows.py --suite fast
|
||||
```
|
||||
|
||||
Workflows in fast suite:
|
||||
- `android_static_analysis`
|
||||
- `python_sast`
|
||||
- `secret_detection`
|
||||
- `gitleaks_detection`
|
||||
- `trufflehog_detection`
|
||||
|
||||
#### Run Full Test Suite
|
||||
|
||||
Tests all workflows (excludes LLM and OSS-Fuzz workflows):
|
||||
|
||||
```bash
|
||||
python scripts/test_workflows.py --suite full
|
||||
```
|
||||
|
||||
Additional workflows in full suite:
|
||||
- `atheris_fuzzing`
|
||||
- `cargo_fuzzing`
|
||||
- `security_assessment`
|
||||
|
||||
#### Run Single Workflow Test
|
||||
|
||||
```bash
|
||||
python scripts/test_workflows.py --workflow python_sast
|
||||
```
|
||||
|
||||
#### Test Platform-Specific Dockerfile
|
||||
|
||||
```bash
|
||||
python scripts/test_workflows.py \
|
||||
--workflow android_static_analysis \
|
||||
--platform linux/amd64
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Matrix Configuration
|
||||
|
||||
The test matrix (`.github/test-matrix.yaml`) defines:
|
||||
|
||||
- Workflow-to-worker mappings
|
||||
- Test projects for each workflow
|
||||
- Required parameters
|
||||
- Expected outcomes
|
||||
- Timeout values
|
||||
- Test suite groupings
|
||||
|
||||
### Example Configuration
|
||||
|
||||
```yaml
|
||||
workflows:
|
||||
python_sast:
|
||||
worker: python
|
||||
test_project: test_projects/vulnerable_app
|
||||
working_directory: test_projects/vulnerable_app
|
||||
parameters: {}
|
||||
timeout: 180
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [python, sast, fast]
|
||||
```
|
||||
|
||||
### Adding a New Workflow Test
|
||||
|
||||
1. Add workflow configuration to `.github/test-matrix.yaml`:
|
||||
|
||||
```yaml
|
||||
workflows:
|
||||
my_new_workflow:
|
||||
worker: python # Which worker runs this workflow
|
||||
test_project: test_projects/my_test
|
||||
working_directory: test_projects/my_test
|
||||
parameters:
|
||||
# Any required parameters
|
||||
severity: "high"
|
||||
timeout: 300
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [python, custom, fast]
|
||||
```
|
||||
|
||||
2. Add to appropriate test suite:
|
||||
|
||||
```yaml
|
||||
test_suites:
|
||||
fast:
|
||||
workflows:
|
||||
- python_sast
|
||||
- my_new_workflow # Add here
|
||||
```
|
||||
|
||||
3. Ensure test project exists with appropriate test cases
|
||||
|
||||
---
|
||||
|
||||
## Platform-Specific Testing
|
||||
|
||||
### Why Platform-Specific Tests?
|
||||
|
||||
Some workers (like Android) have different capabilities on different platforms:
|
||||
|
||||
- **AMD64 (x86_64)**: Full toolchain including MobSF
|
||||
- **ARM64 (Apple Silicon)**: Limited toolchain (MobSF incompatible with Rosetta 2)
|
||||
|
||||
### Platform Detection
|
||||
|
||||
Platform detection happens in `cli/src/fuzzforge_cli/worker_manager.py`:
|
||||
|
||||
```python
|
||||
def _detect_platform(self) -> str:
|
||||
"""Detect current platform for Docker image selection."""
|
||||
machine = platform.machine()
|
||||
system = platform.system()
|
||||
|
||||
# Map to Docker platform identifiers
|
||||
if machine in ['x86_64', 'AMD64']:
|
||||
return 'linux/amd64'
|
||||
elif machine in ['aarch64', 'arm64']:
|
||||
return 'linux/arm64'
|
||||
else:
|
||||
return 'linux/amd64' # Default fallback
|
||||
```
|
||||
|
||||
### Dockerfile Selection
|
||||
|
||||
Workers with `metadata.yaml` can define platform-specific Dockerfiles:
|
||||
|
||||
```yaml
|
||||
# workers/android/metadata.yaml
|
||||
platforms:
|
||||
linux/amd64:
|
||||
dockerfile: Dockerfile.amd64
|
||||
description: "Full Android toolchain with MobSF support"
|
||||
|
||||
linux/arm64:
|
||||
dockerfile: Dockerfile.arm64
|
||||
description: "Android toolchain without MobSF"
|
||||
```
|
||||
|
||||
### Testing Platform Detection
|
||||
|
||||
```bash
|
||||
# Run platform detection unit tests
|
||||
cd cli
|
||||
pytest tests/test_platform_detection.py -v
|
||||
|
||||
# Test with mocked platforms
|
||||
pytest tests/test_platform_detection.py::TestPlatformDetection::test_detect_platform_linux_x86_64 -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CI/CD Testing
|
||||
|
||||
### GitHub Actions Workflows
|
||||
|
||||
#### 1. Main Test Workflow (`.github/workflows/test.yml`)
|
||||
|
||||
Runs on every push and PR:
|
||||
|
||||
- **Worker Validation**: Validates Dockerfiles and metadata
|
||||
- **Docker Image Builds**: Builds only modified workers
|
||||
- **Linting**: Ruff and mypy checks
|
||||
- **Backend Unit Tests**: pytest on Python 3.11 and 3.12
|
||||
|
||||
#### 2. Workflow Integration Tests (`.github/workflows/test-workflows.yml`)
|
||||
|
||||
Runs end-to-end workflow tests:
|
||||
|
||||
- **Platform Detection Tests**: Unit tests for platform detection logic
|
||||
- **Fast Workflow Tests**: Quick smoke tests (runs on every PR)
|
||||
- **Android Platform Tests**: Verifies AMD64 and ARM64 Dockerfile selection
|
||||
- **Full Workflow Tests**: Comprehensive tests (runs on main/master or schedule)
|
||||
|
||||
### Test Triggers
|
||||
|
||||
```yaml
|
||||
# Runs on every push/PR
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, dev, develop, test/** ]
|
||||
pull_request:
|
||||
branches: [ main, master, dev, develop ]
|
||||
|
||||
# Manual trigger with test suite selection
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_suite:
|
||||
type: choice
|
||||
options:
|
||||
- fast
|
||||
- full
|
||||
- platform
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Debugging Test Failures
|
||||
|
||||
### Local Debugging
|
||||
|
||||
#### 1. Check Service Status
|
||||
|
||||
```bash
|
||||
docker ps
|
||||
docker logs fuzzforge-backend
|
||||
docker logs fuzzforge-worker-python
|
||||
```
|
||||
|
||||
#### 2. Run Workflow Manually
|
||||
|
||||
```bash
|
||||
cd test_projects/vulnerable_app
|
||||
ff workflow run python_sast . --wait --no-interactive
|
||||
```
|
||||
|
||||
#### 3. Check Findings
|
||||
|
||||
```bash
|
||||
ff findings list
|
||||
ff findings list python_sast-xxxxx --format json
|
||||
```
|
||||
|
||||
### CI Debugging
|
||||
|
||||
Test workflows automatically collect logs on failure:
|
||||
|
||||
```yaml
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
docker ps -a
|
||||
docker logs fuzzforge-backend --tail 100
|
||||
docker logs fuzzforge-worker-python --tail 50
|
||||
```
|
||||
|
||||
View logs in GitHub Actions:
|
||||
1. Go to failed workflow run
|
||||
2. Click on failed job
|
||||
3. Scroll to "Collect logs on failure" step
|
||||
|
||||
---
|
||||
|
||||
## Writing New Tests
|
||||
|
||||
### Adding a Backend Unit Test
|
||||
|
||||
```python
|
||||
# backend/tests/unit/test_my_feature.py
|
||||
import pytest
|
||||
from toolbox.modules.my_module import my_function
|
||||
|
||||
def test_my_function():
|
||||
result = my_function("input")
|
||||
assert result == "expected_output"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_function():
|
||||
result = await my_async_function()
|
||||
assert result is not None
|
||||
```
|
||||
|
||||
### Adding a CLI Unit Test
|
||||
|
||||
```python
|
||||
# cli/tests/test_my_feature.py
|
||||
import pytest
|
||||
from fuzzforge_cli.my_module import MyClass
|
||||
|
||||
@pytest.fixture
|
||||
def my_instance():
|
||||
return MyClass()
|
||||
|
||||
def test_my_method(my_instance):
|
||||
result = my_instance.my_method()
|
||||
assert result == expected_value
|
||||
```
|
||||
|
||||
### Adding a Platform Detection Test
|
||||
|
||||
```python
|
||||
# cli/tests/test_platform_detection.py
|
||||
from unittest.mock import patch
|
||||
|
||||
def test_detect_platform_linux_x86_64(worker_manager):
|
||||
with patch('platform.machine', return_value='x86_64'), \
|
||||
patch('platform.system', return_value='Linux'):
|
||||
platform = worker_manager._detect_platform()
|
||||
assert platform == 'linux/amd64'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test Coverage
|
||||
|
||||
### Viewing Coverage Reports
|
||||
|
||||
#### Backend Coverage
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
pytest tests/unit/ --cov=toolbox/modules --cov=src --cov-report=html
|
||||
open htmlcov/index.html
|
||||
```
|
||||
|
||||
#### CLI Coverage
|
||||
|
||||
```bash
|
||||
cd cli
|
||||
pytest tests/ --cov=src/fuzzforge_cli --cov-report=html
|
||||
open htmlcov/index.html
|
||||
```
|
||||
|
||||
### Coverage in CI
|
||||
|
||||
Coverage reports are automatically uploaded to Codecov:
|
||||
|
||||
- Backend: `codecov-backend`
|
||||
- CLI Platform Detection: `cli-platform-detection`
|
||||
|
||||
View at: https://codecov.io/gh/FuzzingLabs/fuzzforge_ai
|
||||
|
||||
---
|
||||
|
||||
## Test Best Practices
|
||||
|
||||
### 1. Fast Tests First
|
||||
|
||||
Order tests by execution time:
|
||||
- Unit tests (< 1s each)
|
||||
- Integration tests (< 10s each)
|
||||
- Workflow tests (< 5min each)
|
||||
|
||||
### 2. Use Test Fixtures
|
||||
|
||||
```python
|
||||
@pytest.fixture
|
||||
def temp_project(tmp_path):
|
||||
"""Create temporary test project."""
|
||||
project_dir = tmp_path / "test_project"
|
||||
project_dir.mkdir()
|
||||
# Setup project files
|
||||
return project_dir
|
||||
```
|
||||
|
||||
### 3. Mock External Dependencies
|
||||
|
||||
```python
|
||||
@patch('subprocess.run')
|
||||
def test_docker_command(mock_run):
|
||||
mock_run.return_value = Mock(returncode=0, stdout="success")
|
||||
result = run_docker_command()
|
||||
assert result == "success"
|
||||
```
|
||||
|
||||
### 4. Parametrize Similar Tests
|
||||
|
||||
```python
|
||||
@pytest.mark.parametrize("platform,expected", [
|
||||
("linux/amd64", "Dockerfile.amd64"),
|
||||
("linux/arm64", "Dockerfile.arm64"),
|
||||
])
|
||||
def test_dockerfile_selection(platform, expected):
|
||||
dockerfile = select_dockerfile(platform)
|
||||
assert expected in str(dockerfile)
|
||||
```
|
||||
|
||||
### 5. Tag Tests Appropriately
|
||||
|
||||
```python
|
||||
@pytest.mark.integration
|
||||
def test_full_workflow():
|
||||
# Integration test that requires services
|
||||
pass
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_long_running_operation():
|
||||
# Test that takes > 10 seconds
|
||||
pass
|
||||
```
|
||||
|
||||
Run specific tags:
|
||||
```bash
|
||||
pytest -m "not slow" # Skip slow tests
|
||||
pytest -m integration # Only integration tests
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Continuous Improvement
|
||||
|
||||
### Adding Test Coverage
|
||||
|
||||
1. Identify untested code paths
|
||||
2. Write unit tests for core logic
|
||||
3. Add integration tests for end-to-end flows
|
||||
4. Update test matrix for new workflows
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
1. Use test suites to group tests
|
||||
2. Run fast tests on every commit
|
||||
3. Run slow tests nightly or on main branch
|
||||
4. Parallelize independent tests
|
||||
|
||||
### Monitoring Test Health
|
||||
|
||||
1. Track test execution time trends
|
||||
2. Monitor flaky tests
|
||||
3. Keep coverage above 80%
|
||||
4. Review and update stale tests
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Docker Setup](../how-to/docker-setup.md) - Worker management
|
||||
- [CLI Reference](../reference/cli-reference.md) - CLI commands
|
||||
- [Workflow Guide](../how-to/create-workflow.md) - Creating workflows
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Tests Timeout
|
||||
|
||||
**Symptom**: Workflow tests hang and timeout
|
||||
|
||||
**Solutions**:
|
||||
1. Check if services are running: `docker ps`
|
||||
2. Verify backend is healthy: `docker logs fuzzforge-backend`
|
||||
3. Increase timeout in test matrix
|
||||
4. Check for deadlocks in workflow code
|
||||
|
||||
### Worker Build Failures
|
||||
|
||||
**Symptom**: Docker image build fails in CI
|
||||
|
||||
**Solutions**:
|
||||
1. Test build locally: `docker compose build worker-python`
|
||||
2. Check Dockerfile syntax
|
||||
3. Verify base image is accessible
|
||||
4. Review build logs for specific errors
|
||||
|
||||
### Platform Detection Failures
|
||||
|
||||
**Symptom**: Wrong Dockerfile selected on ARM64
|
||||
|
||||
**Solutions**:
|
||||
1. Verify metadata.yaml syntax
|
||||
2. Check platform detection logic
|
||||
3. Test locally with: `python -c "import platform; print(platform.machine())"`
|
||||
4. Review WorkerManager._detect_platform() logic
|
||||
|
||||
### SARIF Export Validation Fails
|
||||
|
||||
**Symptom**: Workflow completes but SARIF validation fails
|
||||
|
||||
**Solutions**:
|
||||
1. Check SARIF file exists: `ls -la test-*.sarif`
|
||||
2. Validate JSON syntax: `jq . test-*.sarif`
|
||||
3. Verify SARIF schema: Must have `version` and `runs` fields
|
||||
4. Check workflow SARIF export logic
|
||||
|
||||
---
|
||||
|
||||
**Questions?** Open an issue or consult the [development discussions](https://github.com/FuzzingLabs/fuzzforge_ai/discussions).
|
||||
@@ -110,7 +110,22 @@ fuzzforge workflow run secret_detection ./codebase
|
||||
|
||||
### Manual Worker Management
|
||||
|
||||
Start specific workers when needed:
|
||||
FuzzForge CLI provides convenient commands for managing workers:
|
||||
|
||||
```bash
|
||||
# List all workers and their status
|
||||
ff worker list
|
||||
ff worker list --all # Include stopped workers
|
||||
|
||||
# Start a specific worker
|
||||
ff worker start python
|
||||
ff worker start android --build # Rebuild before starting
|
||||
|
||||
# Stop all workers
|
||||
ff worker stop
|
||||
```
|
||||
|
||||
You can also use Docker commands directly:
|
||||
|
||||
```bash
|
||||
# Start a single worker
|
||||
@@ -123,6 +138,33 @@ docker compose --profile workers up -d
|
||||
docker stop fuzzforge-worker-ossfuzz
|
||||
```
|
||||
|
||||
### Stopping Workers Properly
|
||||
|
||||
The easiest way to stop workers is using the CLI:
|
||||
|
||||
```bash
|
||||
# Stop all running workers (recommended)
|
||||
ff worker stop
|
||||
```
|
||||
|
||||
This command safely stops all worker containers without affecting core services.
|
||||
|
||||
Alternatively, you can use Docker commands:
|
||||
|
||||
```bash
|
||||
# Stop individual worker
|
||||
docker stop fuzzforge-worker-python
|
||||
|
||||
# Stop all workers using docker compose
|
||||
# Note: This requires the --profile flag because workers are in profiles
|
||||
docker compose down --profile workers
|
||||
```
|
||||
|
||||
**Important:** Workers use Docker Compose profiles to prevent auto-starting. When using Docker commands directly:
|
||||
- `docker compose down` (without `--profile workers`) does NOT stop workers
|
||||
- Workers remain running unless explicitly stopped with the profile flag or `docker stop`
|
||||
- Use `ff worker stop` for the safest option that won't affect core services
|
||||
|
||||
### Resource Comparison
|
||||
|
||||
| Command | Workers Started | RAM Usage |
|
||||
@@ -171,7 +213,7 @@ FuzzForge requires `volumes/env/.env` to start. This file contains API keys and
|
||||
|
||||
```bash
|
||||
# Copy the example file
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
# Edit to add your API keys (if using AI features)
|
||||
nano volumes/env/.env
|
||||
|
||||
179
docs/docs/how-to/litellm-hot-swap.md
Normal file
179
docs/docs/how-to/litellm-hot-swap.md
Normal file
@@ -0,0 +1,179 @@
|
||||
---
|
||||
title: "Hot-Swap LiteLLM Models"
|
||||
description: "Register OpenAI and Anthropic models with the bundled LiteLLM proxy and switch them on the task agent without downtime."
|
||||
---
|
||||
|
||||
LiteLLM sits between the task agent and upstream providers, so every model change
|
||||
is just an API call. This guide walks through registering OpenAI and Anthropic
|
||||
models, updating the virtual key, and exercising the A2A hot-swap flow.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- `docker compose up llm-proxy llm-proxy-db task-agent`
|
||||
- Provider secrets in `volumes/env/.env`:
|
||||
- `LITELLM_OPENAI_API_KEY`
|
||||
- `LITELLM_ANTHROPIC_API_KEY`
|
||||
- Master key (`LITELLM_MASTER_KEY`) and task-agent virtual key (auto-generated
|
||||
during bootstrap)
|
||||
|
||||
> UI access uses `UI_USERNAME` / `UI_PASSWORD` (defaults: `fuzzforge` /
|
||||
> `fuzzforge123`). Change them by exporting new values before running compose.
|
||||
|
||||
## Register Provider Models
|
||||
|
||||
Use the admin API to register the models the proxy should expose. The snippet
|
||||
below creates aliases for OpenAI `gpt-5`, `gpt-5-mini`, and Anthropic
|
||||
`claude-sonnet-4-5`.
|
||||
|
||||
```bash
|
||||
MASTER_KEY=$(awk -F= '$1=="LITELLM_MASTER_KEY"{print $2}' volumes/env/.env)
|
||||
export OPENAI_API_KEY=$(awk -F= '$1=="OPENAI_API_KEY"{print $2}' volumes/env/.env)
|
||||
python - <<'PY'
|
||||
import os, requests
|
||||
master = os.environ['MASTER_KEY'].strip()
|
||||
base = 'http://localhost:10999'
|
||||
models = [
|
||||
{
|
||||
"model_name": "openai/gpt-5",
|
||||
"litellm_params": {
|
||||
"model": "gpt-5",
|
||||
"custom_llm_provider": "openai",
|
||||
"api_key": "os.environ/LITELLM_OPENAI_API_KEY"
|
||||
},
|
||||
"model_info": {
|
||||
"provider": "openai",
|
||||
"description": "OpenAI GPT-5"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model_name": "openai/gpt-5-mini",
|
||||
"litellm_params": {
|
||||
"model": "gpt-5-mini",
|
||||
"custom_llm_provider": "openai",
|
||||
"api_key": "os.environ/LITELLM_OPENAI_API_KEY"
|
||||
},
|
||||
"model_info": {
|
||||
"provider": "openai",
|
||||
"description": "OpenAI GPT-5 mini"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model_name": "anthropic/claude-sonnet-4-5",
|
||||
"litellm_params": {
|
||||
"model": "claude-sonnet-4-5",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"api_key": "os.environ/LITELLM_ANTHROPIC_API_KEY"
|
||||
},
|
||||
"model_info": {
|
||||
"provider": "anthropic",
|
||||
"description": "Anthropic Claude Sonnet 4.5"
|
||||
}
|
||||
}
|
||||
]
|
||||
for payload in models:
|
||||
resp = requests.post(
|
||||
f"{base}/model/new",
|
||||
headers={"Authorization": f"Bearer {master}", "Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=60,
|
||||
)
|
||||
if resp.status_code not in (200, 201, 409):
|
||||
raise SystemExit(f"Failed to register {payload['model_name']}: {resp.status_code} {resp.text}")
|
||||
print(payload['model_name'], '=>', resp.status_code)
|
||||
PY
|
||||
```
|
||||
|
||||
Each entry stores the upstream secret by reference (`os.environ/...`) so the
|
||||
raw API key never leaves the container environment.
|
||||
|
||||
## Relax Virtual Key Model Restrictions
|
||||
|
||||
Let the agent key call every model on the proxy:
|
||||
|
||||
```bash
|
||||
MASTER_KEY=$(awk -F= '$1=="LITELLM_MASTER_KEY"{print $2}' volumes/env/.env)
|
||||
VK=$(awk -F= '$1=="OPENAI_API_KEY"{print $2}' volumes/env/.env)
|
||||
python - <<'PY'
|
||||
import os, requests, json
|
||||
resp = requests.post(
|
||||
'http://localhost:10999/key/update',
|
||||
headers={
|
||||
'Authorization': f"Bearer {os.environ['MASTER_KEY'].strip()}",
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
json={'key': os.environ['VK'].strip(), 'models': []},
|
||||
timeout=60,
|
||||
)
|
||||
print(json.dumps(resp.json(), indent=2))
|
||||
PY
|
||||
```
|
||||
|
||||
Restart the task agent so it sees the refreshed key:
|
||||
|
||||
```bash
|
||||
docker compose restart task-agent
|
||||
```
|
||||
|
||||
## Hot-Swap With The A2A Helper
|
||||
|
||||
Switch models without restarting the service:
|
||||
|
||||
```bash
|
||||
# Ensure the CLI reads the latest virtual key
|
||||
export OPENAI_API_KEY=$(awk -F= '$1=="OPENAI_API_KEY"{print $2}' volumes/env/.env)
|
||||
|
||||
# OpenAI gpt-5 alias
|
||||
python ai/agents/task_agent/a2a_hot_swap.py \
|
||||
--url http://localhost:10900/a2a/litellm_agent \
|
||||
--model openai gpt-5 \
|
||||
--context switch-demo
|
||||
|
||||
# Confirm the response comes from the new model
|
||||
python ai/agents/task_agent/a2a_hot_swap.py \
|
||||
--url http://localhost:10900/a2a/litellm_agent \
|
||||
--message "Which model am I using?" \
|
||||
--context switch-demo
|
||||
|
||||
# Swap to gpt-5-mini
|
||||
python ai/agents/task_agent/a2a_hot_swap.py --url http://localhost:10900/a2a/litellm_agent --model openai gpt-5-mini --context switch-demo
|
||||
|
||||
# Swap to Anthropic Claude Sonnet 4.5
|
||||
python ai/agents/task_agent/a2a_hot_swap.py --url http://localhost:10900/a2a/litellm_agent --model anthropic claude-sonnet-4-5 --context switch-demo
|
||||
```
|
||||
|
||||
> Each invocation reuses the same conversation context (`switch-demo`) so you
|
||||
> can confirm the active provider by asking follow-up questions.
|
||||
|
||||
## Resetting The Proxy (Optional)
|
||||
|
||||
To wipe the LiteLLM state and rerun bootstrap:
|
||||
|
||||
```bash
|
||||
docker compose down llm-proxy llm-proxy-db llm-proxy-bootstrap
|
||||
|
||||
docker volume rm fuzzforge_litellm_proxy_data fuzzforge_litellm_proxy_db
|
||||
|
||||
docker compose up -d llm-proxy-db llm-proxy
|
||||
```
|
||||
|
||||
After the proxy is healthy, rerun the registration script and key update. The
|
||||
bootstrap container mirrors secrets into `.env.litellm` and reissues the task
|
||||
agent key automatically.
|
||||
|
||||
## How The Pieces Fit Together
|
||||
|
||||
1. **LiteLLM Proxy** exposes OpenAI-compatible routes and stores provider
|
||||
metadata in Postgres.
|
||||
2. **Bootstrap Container** waits for `/health/liveliness`, mirrors secrets into
|
||||
`.env.litellm`, registers any models you script, and keeps the virtual key in
|
||||
sync with the discovered model list.
|
||||
3. **Task Agent** calls the proxy via `FF_LLM_PROXY_BASE_URL`. The hot-swap tool
|
||||
updates the agent’s runtime configuration, so switching providers is just a
|
||||
control message.
|
||||
4. **Virtual Keys** carry quotas and allowed models. Setting the `models` array
|
||||
to `[]` lets the key use anything registered on the proxy.
|
||||
|
||||
Keep the master key and generated virtual keys somewhere safe—they grant full
|
||||
admin and agent access respectively. When you add a new provider (e.g., Ollama)
|
||||
just register the model via `/model/new`, update the key if needed, and repeat
|
||||
the hot-swap steps.
|
||||
194
docs/docs/how-to/llm-proxy.md
Normal file
194
docs/docs/how-to/llm-proxy.md
Normal file
@@ -0,0 +1,194 @@
|
||||
---
|
||||
title: "Run the LLM Proxy"
|
||||
description: "Run the LiteLLM gateway that ships with FuzzForge and connect it to the task agent."
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
FuzzForge routes every LLM request through a LiteLLM proxy so that usage can be
|
||||
metered, priced, and rate limited per user. Docker Compose starts the proxy in a
|
||||
hardened container, while a bootstrap job seeds upstream provider secrets and
|
||||
issues a virtual key for the task agent automatically.
|
||||
|
||||
LiteLLM exposes the OpenAI-compatible APIs (`/v1/*`) plus a rich admin UI. All
|
||||
traffic stays on your network and upstream credentials never leave the proxy
|
||||
container.
|
||||
|
||||
## Before You Start
|
||||
|
||||
1. Copy `volumes/env/.env.template` to `volumes/env/.env` and set the basics:
|
||||
- `LITELLM_MASTER_KEY` — admin token used to manage the proxy
|
||||
- `LITELLM_SALT_KEY` — random string used to encrypt provider credentials
|
||||
- Provider secrets under `LITELLM_<PROVIDER>_API_KEY` (for example
|
||||
`LITELLM_OPENAI_API_KEY`)
|
||||
- Leave `OPENAI_API_KEY=sk-proxy-default`; the bootstrap job replaces it with a
|
||||
LiteLLM-issued virtual key
|
||||
2. When running tools outside Docker, change `FF_LLM_PROXY_BASE_URL` to the
|
||||
published host port (`http://localhost:10999`). Inside Docker the default
|
||||
value `http://llm-proxy:4000` already resolves to the container.
|
||||
|
||||
## Start the Proxy
|
||||
|
||||
```bash
|
||||
docker compose up llm-proxy
|
||||
```
|
||||
|
||||
The service publishes two things:
|
||||
|
||||
- HTTP API + admin UI on `http://localhost:10999`
|
||||
- Persistent SQLite state inside the named volume
|
||||
`fuzzforge_litellm_proxy_data`
|
||||
|
||||
The UI login uses the `UI_USERNAME` / `UI_PASSWORD` pair (defaults to
|
||||
`fuzzforge` / `fuzzforge123`). To change them, set the environment variables
|
||||
before you run `docker compose up`:
|
||||
|
||||
```bash
|
||||
export UI_USERNAME=myadmin
|
||||
export UI_PASSWORD=super-secret
|
||||
docker compose up llm-proxy
|
||||
```
|
||||
|
||||
You can also edit the values directly in `docker-compose.yml` if you prefer to
|
||||
check them into a different secrets manager.
|
||||
|
||||
Proxy-wide settings now live in `volumes/litellm/proxy_config.yaml`. By
|
||||
default it enables `store_model_in_db` and `store_prompts_in_spend_logs`, which
|
||||
lets the UI display request/response payloads for new calls. Update this file
|
||||
if you need additional LiteLLM options and restart the `llm-proxy` container.
|
||||
|
||||
LiteLLM's health endpoint lives at `/health/liveliness`. You can verify it from
|
||||
another terminal:
|
||||
|
||||
```bash
|
||||
curl http://localhost:10999/health/liveliness
|
||||
```
|
||||
|
||||
## What the Bootstrapper Does
|
||||
|
||||
During startup the `llm-proxy-bootstrap` container performs three actions:
|
||||
|
||||
1. **Wait for the proxy** — Blocks until `/health/liveliness` becomes healthy.
|
||||
2. **Mirror provider secrets** — Reads `volumes/env/.env` and writes any
|
||||
`LITELLM_*_API_KEY` values into `volumes/env/.env.litellm`. The file is
|
||||
created automatically on first boot; if you delete it, bootstrap will
|
||||
recreate it and the proxy continues to read secrets from `.env`.
|
||||
3. **Issue the default virtual key** — Calls `/key/generate` with the master key
|
||||
and persists the generated token back into `volumes/env/.env` (replacing the
|
||||
`sk-proxy-default` placeholder). The key is scoped to
|
||||
`LITELLM_DEFAULT_MODELS` when that variable is set; otherwise it uses the
|
||||
model from `LITELLM_MODEL`.
|
||||
|
||||
The sequence is idempotent. Existing provider secrets and virtual keys are
|
||||
reused on subsequent runs, and the allowed-model list is refreshed via
|
||||
`/key/update` if you change the defaults.
|
||||
|
||||
## Managing Virtual Keys
|
||||
|
||||
LiteLLM keys act as per-user credentials. The default key, named
|
||||
`task-agent default`, is created automatically for the task agent. You can issue
|
||||
more keys for teammates or CI jobs with the same management API:
|
||||
|
||||
```bash
|
||||
curl http://localhost:10999/key/generate \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"key_alias": "demo-user",
|
||||
"user_id": "demo",
|
||||
"models": ["openai/gpt-4o-mini"],
|
||||
"duration": "30d",
|
||||
"max_budget": 50,
|
||||
"metadata": {"team": "sandbox"}
|
||||
}'
|
||||
```
|
||||
|
||||
Use `/key/update` to adjust budgets or the allowed-model list on existing keys:
|
||||
|
||||
```bash
|
||||
curl http://localhost:10999/key/update \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"key": "sk-...",
|
||||
"models": ["openai/*", "anthropic/*"],
|
||||
"max_budget": 100
|
||||
}'
|
||||
```
|
||||
|
||||
The admin UI (navigate to `http://localhost:10999/ui`) provides equivalent
|
||||
controls for creating keys, routing models, auditing spend, and exporting logs.
|
||||
|
||||
## Wiring the Task Agent
|
||||
|
||||
The task agent already expects to talk to the proxy. Confirm these values in
|
||||
`volumes/env/.env` before launching the stack:
|
||||
|
||||
```bash
|
||||
FF_LLM_PROXY_BASE_URL=http://llm-proxy:4000 # or http://localhost:10999 when outside Docker
|
||||
OPENAI_API_KEY=<virtual key created by bootstrap>
|
||||
LITELLM_MODEL=openai/gpt-5
|
||||
LITELLM_PROVIDER=openai
|
||||
```
|
||||
|
||||
Restart the agent container after changing environment variables so the process
|
||||
picks up the updates.
|
||||
|
||||
To validate the integration end to end, call the proxy directly:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:10999/v1/chat/completions \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "Proxy health check"}]
|
||||
}'
|
||||
```
|
||||
|
||||
A JSON response indicates the proxy can reach your upstream provider using the
|
||||
mirrored secrets.
|
||||
|
||||
## Local Runtimes (Ollama, etc.)
|
||||
|
||||
LiteLLM supports non-hosted providers as well. To route requests to a local
|
||||
runtime such as Ollama:
|
||||
|
||||
1. Set the appropriate provider key in the env file
|
||||
(for Ollama, point LiteLLM at `OLLAMA_API_BASE` inside the container).
|
||||
2. Add the passthrough model either from the UI (**Models → Add Model**) or
|
||||
by calling `/model/new` with the master key.
|
||||
3. Update `LITELLM_DEFAULT_MODELS` (and regenerate the virtual key if you want
|
||||
the default key to include it).
|
||||
|
||||
The task agent keeps using the same OpenAI-compatible surface while LiteLLM
|
||||
handles the translation to your runtime.
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Explore [LiteLLM's documentation](https://docs.litellm.ai/docs/simple_proxy)
|
||||
for advanced routing, cost controls, and observability hooks.
|
||||
- Configure Slack/Prometheus integrations from the UI to monitor usage.
|
||||
- Rotate the master key periodically and store it in your secrets manager, as it
|
||||
grants full admin access to the proxy.
|
||||
|
||||
## Observability
|
||||
|
||||
LiteLLM ships with OpenTelemetry hooks for traces and metrics. This repository
|
||||
already includes an OTLP collector (`otel-collector` service) and mounts a
|
||||
default configuration that forwards traces to standard output. To wire it up:
|
||||
|
||||
1. Edit `volumes/otel/collector-config.yaml` if you want to forward to Jaeger,
|
||||
Datadog, etc. The initial config uses the logging exporter so you can see
|
||||
spans immediately via `docker compose logs -f otel-collector`.
|
||||
2. Customize `volumes/litellm/proxy_config.yaml` if you need additional
|
||||
callbacks; `general_settings.otel: true` and `litellm_settings.callbacks:
|
||||
["otel"]` are already present so no extra code changes are required.
|
||||
3. (Optional) Override `OTEL_EXPORTER_OTLP_*` environment variables in
|
||||
`docker-compose.yml` or your shell to point at a remote collector.
|
||||
|
||||
After updating the configs, run `docker compose up -d otel-collector llm-proxy`
|
||||
and generate a request (for example, trigger `ff workflow run llm_analysis`).
|
||||
New traces will show up in the collector logs or whichever backend you
|
||||
configured. See the official LiteLLM guide for advanced exporter options:
|
||||
https://docs.litellm.ai/docs/observability/opentelemetry_integration.
|
||||
@@ -33,7 +33,7 @@ The required `volumes/env/.env` file is missing. Docker Compose needs this file
|
||||
**How to fix:**
|
||||
```bash
|
||||
# Create the environment file from the template
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
# Restart Docker Compose
|
||||
docker compose -f docker-compose.yml down
|
||||
|
||||
616
docs/docs/reference/cli-reference.md
Normal file
616
docs/docs/reference/cli-reference.md
Normal file
@@ -0,0 +1,616 @@
|
||||
# FuzzForge CLI Reference
|
||||
|
||||
Complete reference for the FuzzForge CLI (`ff` command). Use this as your quick lookup for all commands, options, and examples.
|
||||
|
||||
---
|
||||
|
||||
## Global Options
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| `--help`, `-h` | Show help message |
|
||||
| `--version`, `-v` | Show version information |
|
||||
|
||||
---
|
||||
|
||||
## Core Commands
|
||||
|
||||
### `ff init`
|
||||
|
||||
Initialize a new FuzzForge project in the current directory.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff init [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--name`, `-n` — Project name (defaults to current directory name)
|
||||
- `--api-url`, `-u` — FuzzForge API URL (defaults to http://localhost:8000)
|
||||
- `--force`, `-f` — Force initialization even if project already exists
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff init # Initialize with defaults
|
||||
ff init --name my-project # Set custom project name
|
||||
ff init --api-url http://prod:8000 # Use custom API URL
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `ff status`
|
||||
|
||||
Show project and latest execution status.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff status
|
||||
```
|
||||
|
||||
**Example Output:**
|
||||
```
|
||||
📊 Project Status
|
||||
Project: my-security-project
|
||||
API URL: http://localhost:8000
|
||||
|
||||
Latest Execution:
|
||||
Run ID: security_scan-a1b2c3
|
||||
Workflow: security_assessment
|
||||
Status: COMPLETED
|
||||
Started: 2 hours ago
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `ff config`
|
||||
|
||||
Manage project configuration.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff config # Show all config
|
||||
ff config <key> # Get specific value
|
||||
ff config <key> <value> # Set value
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff config # Display all settings
|
||||
ff config api_url # Get API URL
|
||||
ff config api_url http://prod:8000 # Set API URL
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `ff clean`
|
||||
|
||||
Clean old execution data and findings.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff clean [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--days`, `-d` — Remove data older than this many days (default: 90)
|
||||
- `--dry-run` — Show what would be deleted without deleting
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff clean # Clean data older than 90 days
|
||||
ff clean --days 30 # Clean data older than 30 days
|
||||
ff clean --dry-run # Preview what would be deleted
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Workflow Commands
|
||||
|
||||
### `ff workflows`
|
||||
|
||||
Browse and list available workflows.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff workflows [COMMAND]
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
- `list` — List all available workflows
|
||||
- `info <workflow>` — Show detailed workflow information
|
||||
- `params <workflow>` — Show workflow parameters
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff workflows list # List all workflows
|
||||
ff workflows info python_sast # Show workflow details
|
||||
ff workflows params python_sast # Show parameters
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `ff workflow`
|
||||
|
||||
Execute and manage individual workflows.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff workflow <COMMAND>
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
|
||||
#### `ff workflow run`
|
||||
|
||||
Execute a security testing workflow.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff workflow run <workflow> <target> [params...] [OPTIONS]
|
||||
```
|
||||
|
||||
**Arguments:**
|
||||
- `<workflow>` — Workflow name
|
||||
- `<target>` — Target path to analyze
|
||||
- `[params...]` — Parameters as `key=value` pairs
|
||||
|
||||
**Options:**
|
||||
- `--param-file`, `-f` — JSON file containing workflow parameters
|
||||
- `--timeout`, `-t` — Execution timeout in seconds
|
||||
- `--interactive` / `--no-interactive`, `-i` / `-n` — Interactive parameter input (default: interactive)
|
||||
- `--wait`, `-w` — Wait for execution to complete
|
||||
- `--live`, `-l` — Start live monitoring after execution
|
||||
- `--auto-start` / `--no-auto-start` — Automatically start required worker
|
||||
- `--auto-stop` / `--no-auto-stop` — Automatically stop worker after completion
|
||||
- `--fail-on` — Fail build if findings match SARIF level (error, warning, note, info, all, none)
|
||||
- `--export-sarif` — Export SARIF results to file after completion
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
# Basic workflow execution
|
||||
ff workflow run python_sast ./project
|
||||
|
||||
# With parameters
|
||||
ff workflow run python_sast ./project check_secrets=true
|
||||
|
||||
# CI/CD integration - fail on errors
|
||||
ff workflow run python_sast ./project --wait --no-interactive \
|
||||
--fail-on error --export-sarif results.sarif
|
||||
|
||||
# With parameter file
|
||||
ff workflow run python_sast ./project --param-file config.json
|
||||
|
||||
# Live monitoring for fuzzing
|
||||
ff workflow run atheris_fuzzing ./project --live
|
||||
```
|
||||
|
||||
#### `ff workflow status`
|
||||
|
||||
Check status of latest or specific workflow execution.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff workflow status [run_id]
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff workflow status # Show latest execution status
|
||||
ff workflow status python_sast-abc123 # Show specific execution
|
||||
```
|
||||
|
||||
#### `ff workflow history`
|
||||
|
||||
Show execution history.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff workflow history [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--limit`, `-l` — Number of executions to show (default: 10)
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
ff workflow history --limit 20
|
||||
```
|
||||
|
||||
#### `ff workflow retry`
|
||||
|
||||
Retry a failed workflow execution.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff workflow retry <run_id>
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
ff workflow retry python_sast-abc123
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Finding Commands
|
||||
|
||||
### `ff findings`
|
||||
|
||||
Browse all findings across executions.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff findings [COMMAND]
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
|
||||
#### `ff findings list`
|
||||
|
||||
List findings from a specific run.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff findings list [run_id] [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--format` — Output format: table, json, sarif (default: table)
|
||||
- `--save` — Save findings to file
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff findings list # Show latest findings
|
||||
ff findings list python_sast-abc123 # Show specific run
|
||||
ff findings list --format json # JSON output
|
||||
ff findings list --format sarif --save # Export SARIF
|
||||
```
|
||||
|
||||
#### `ff findings export`
|
||||
|
||||
Export findings to various formats.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff findings export <run_id> [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--format` — Output format: json, sarif, csv
|
||||
- `--output`, `-o` — Output file path
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
ff findings export python_sast-abc123 --format sarif --output results.sarif
|
||||
```
|
||||
|
||||
#### `ff findings history`
|
||||
|
||||
Show finding history across multiple runs.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff findings history [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--limit`, `-l` — Number of runs to include (default: 10)
|
||||
|
||||
---
|
||||
|
||||
### `ff finding`
|
||||
|
||||
View and analyze individual findings.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff finding [id] # Show latest or specific finding
|
||||
ff finding show <run_id> --rule <rule> # Show specific finding detail
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff finding # Show latest finding
|
||||
ff finding python_sast-abc123 # Show specific run findings
|
||||
ff finding show python_sast-abc123 --rule f2cf5e3e # Show specific finding
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Worker Management Commands
|
||||
|
||||
### `ff worker`
|
||||
|
||||
Manage Temporal workers for workflow execution.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff worker <COMMAND>
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
|
||||
#### `ff worker list`
|
||||
|
||||
List FuzzForge workers and their status.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff worker list [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--all`, `-a` — Show all workers (including stopped)
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff worker list # Show running workers
|
||||
ff worker list --all # Show all workers
|
||||
```
|
||||
|
||||
**Example Output:**
|
||||
```
|
||||
FuzzForge Workers
|
||||
┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┓
|
||||
┃ Worker ┃ Status ┃ Uptime ┃
|
||||
┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━┩
|
||||
│ android │ ● Running │ 5 minutes ago │
|
||||
│ python │ ● Running │ 10 minutes ago │
|
||||
└─────────┴───────────┴────────────────┘
|
||||
|
||||
✅ 2 worker(s) running
|
||||
```
|
||||
|
||||
#### `ff worker start`
|
||||
|
||||
Start a specific worker.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff worker start <name> [OPTIONS]
|
||||
```
|
||||
|
||||
**Arguments:**
|
||||
- `<name>` — Worker name (e.g., python, android, rust, secrets)
|
||||
|
||||
**Options:**
|
||||
- `--build` — Rebuild worker image before starting
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff worker start python # Start Python worker
|
||||
ff worker start android --build # Rebuild and start Android worker
|
||||
```
|
||||
|
||||
**Available Workers:**
|
||||
- `python` — Python security analysis and fuzzing
|
||||
- `android` — Android APK analysis
|
||||
- `rust` — Rust fuzzing and analysis
|
||||
- `secrets` — Secret detection workflows
|
||||
- `ossfuzz` — OSS-Fuzz integration
|
||||
|
||||
#### `ff worker stop`
|
||||
|
||||
Stop all running FuzzForge workers.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff worker stop [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
- `--all` — Stop all workers (default behavior, flag for clarity)
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
ff worker stop
|
||||
```
|
||||
|
||||
**Note:** This command stops only worker containers, leaving core services (backend, temporal, minio) running.
|
||||
|
||||
---
|
||||
|
||||
## Monitoring Commands
|
||||
|
||||
### `ff monitor`
|
||||
|
||||
Real-time monitoring for running workflows.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff monitor [COMMAND]
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
- `live <run_id>` — Live monitoring for a specific execution
|
||||
- `stats <run_id>` — Show statistics for fuzzing workflows
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff monitor live atheris-abc123 # Monitor fuzzing campaign
|
||||
ff monitor stats atheris-abc123 # Show fuzzing statistics
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## AI Integration Commands
|
||||
|
||||
### `ff ai`
|
||||
|
||||
AI-powered analysis and assistance.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff ai [COMMAND]
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
- `analyze <run_id>` — Analyze findings with AI
|
||||
- `explain <finding_id>` — Get AI explanation of a finding
|
||||
- `remediate <finding_id>` — Get remediation suggestions
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff ai analyze python_sast-abc123 # Analyze all findings
|
||||
ff ai explain python_sast-abc123:finding1 # Explain specific finding
|
||||
ff ai remediate python_sast-abc123:finding1 # Get fix suggestions
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Knowledge Ingestion Commands
|
||||
|
||||
### `ff ingest`
|
||||
|
||||
Ingest knowledge into the AI knowledge base.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
ff ingest [COMMAND]
|
||||
```
|
||||
|
||||
**Subcommands:**
|
||||
- `file <path>` — Ingest a file
|
||||
- `directory <path>` — Ingest directory contents
|
||||
- `workflow <workflow_name>` — Ingest workflow documentation
|
||||
|
||||
**Examples:**
|
||||
```bash
|
||||
ff ingest file ./docs/security.md # Ingest single file
|
||||
ff ingest directory ./docs # Ingest directory
|
||||
ff ingest workflow python_sast # Ingest workflow docs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common Workflow Examples
|
||||
|
||||
### CI/CD Integration
|
||||
|
||||
```bash
|
||||
# Run security scan in CI, fail on errors
|
||||
ff workflow run python_sast . \
|
||||
--wait \
|
||||
--no-interactive \
|
||||
--fail-on error \
|
||||
--export-sarif results.sarif
|
||||
```
|
||||
|
||||
### Local Development
|
||||
|
||||
```bash
|
||||
# Quick security check
|
||||
ff workflow run python_sast ./my-code
|
||||
|
||||
# Check specific file types
|
||||
ff workflow run python_sast . file_extensions='[".py",".js"]'
|
||||
|
||||
# Interactive parameter configuration
|
||||
ff workflow run python_sast . --interactive
|
||||
```
|
||||
|
||||
### Fuzzing Workflows
|
||||
|
||||
```bash
|
||||
# Start fuzzing with live monitoring
|
||||
ff workflow run atheris_fuzzing ./project --live
|
||||
|
||||
# Long-running fuzzing campaign
|
||||
ff workflow run ossfuzz_campaign ./project \
|
||||
--auto-start \
|
||||
duration=3600 \
|
||||
--live
|
||||
```
|
||||
|
||||
### Worker Management
|
||||
|
||||
```bash
|
||||
# Check which workers are running
|
||||
ff worker list
|
||||
|
||||
# Start needed worker manually
|
||||
ff worker start python --build
|
||||
|
||||
# Stop all workers when done
|
||||
ff worker stop
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration Files
|
||||
|
||||
### Project Config (`.fuzzforge/config.json`)
|
||||
|
||||
```json
|
||||
{
|
||||
"project_name": "my-security-project",
|
||||
"api_url": "http://localhost:8000",
|
||||
"default_workflow": "python_sast",
|
||||
"auto_start_workers": true,
|
||||
"auto_stop_workers": false
|
||||
}
|
||||
```
|
||||
|
||||
### Parameter File Example
|
||||
|
||||
```json
|
||||
{
|
||||
"check_secrets": true,
|
||||
"file_extensions": [".py", ".js", ".go"],
|
||||
"severity_threshold": "medium",
|
||||
"exclude_patterns": ["**/test/**", "**/vendor/**"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Exit Codes
|
||||
|
||||
| Code | Meaning |
|
||||
|------|---------|
|
||||
| 0 | Success |
|
||||
| 1 | General error |
|
||||
| 2 | Findings matched `--fail-on` criteria |
|
||||
| 3 | Worker startup failed |
|
||||
| 4 | Workflow execution failed |
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `FUZZFORGE_API_URL` | Backend API URL | http://localhost:8000 |
|
||||
| `FUZZFORGE_ROOT` | FuzzForge installation directory | Auto-detected |
|
||||
| `FUZZFORGE_DEBUG` | Enable debug logging | false |
|
||||
|
||||
---
|
||||
|
||||
## Tips and Best Practices
|
||||
|
||||
1. **Use `--no-interactive` in CI/CD** — Prevents prompts that would hang automated pipelines
|
||||
2. **Use `--fail-on` for quality gates** — Fail builds based on finding severity
|
||||
3. **Export SARIF for tool integration** — Most security tools support SARIF format
|
||||
4. **Let workflows auto-start workers** — More efficient than manually managing workers
|
||||
5. **Use `--wait` with `--export-sarif`** — Ensures results are available before export
|
||||
6. **Check `ff worker list` regularly** — Helps manage system resources
|
||||
7. **Use parameter files for complex configs** — Easier to version control and reuse
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Docker Setup](../how-to/docker-setup.md) — Worker management and Docker configuration
|
||||
- [Getting Started](../tutorial/getting-started.md) — Complete setup guide
|
||||
- [Workflow Guide](../how-to/create-workflow.md) — Detailed workflow documentation
|
||||
- [CI/CD Integration](../how-to/cicd-integration.md) — CI/CD setup examples
|
||||
|
||||
---
|
||||
|
||||
**Need Help?**
|
||||
|
||||
```bash
|
||||
ff --help # General help
|
||||
ff workflow run --help # Command-specific help
|
||||
ff worker --help # Worker management help
|
||||
```
|
||||
@@ -28,7 +28,7 @@ cd fuzzforge_ai
|
||||
Create the environment configuration file:
|
||||
|
||||
```bash
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
```
|
||||
|
||||
This file is required for FuzzForge to start. You can leave it with default values if you're only using basic workflows.
|
||||
|
||||
@@ -89,7 +89,7 @@ Technical reference materials and specifications.
|
||||
Before starting FuzzForge, you **must** create the environment configuration file:
|
||||
|
||||
```bash
|
||||
cp volumes/env/.env.example volumes/env/.env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
```
|
||||
|
||||
Docker Compose will fail without this file. You can leave it with default values if you're only using basic workflows (no AI features).
|
||||
|
||||
381
scripts/test_workflows.py
Executable file
381
scripts/test_workflows.py
Executable file
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Automated workflow testing script for FuzzForge.
|
||||
|
||||
This script reads the test matrix configuration and executes workflows
|
||||
to validate end-to-end functionality, SARIF export, and platform-specific
|
||||
Dockerfile selection.
|
||||
|
||||
Usage:
|
||||
python scripts/test_workflows.py --suite fast
|
||||
python scripts/test_workflows.py --workflow python_sast
|
||||
python scripts/test_workflows.py --workflow android_static_analysis --platform linux/amd64
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("Error: PyYAML is required. Install with: pip install pyyaml")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowTestResult:
|
||||
"""Result of a workflow test execution."""
|
||||
workflow_name: str
|
||||
success: bool
|
||||
duration: float
|
||||
status: Optional[str] = None
|
||||
run_id: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
findings_count: Optional[int] = None
|
||||
sarif_exported: bool = False
|
||||
|
||||
|
||||
class WorkflowTester:
|
||||
"""Executes and validates FuzzForge workflows."""
|
||||
|
||||
def __init__(self, matrix_file: Path, root_dir: Path):
|
||||
self.matrix_file = matrix_file
|
||||
self.root_dir = root_dir
|
||||
self.matrix = self._load_matrix()
|
||||
self.results: List[WorkflowTestResult] = []
|
||||
|
||||
def _load_matrix(self) -> Dict[str, Any]:
|
||||
"""Load test matrix configuration."""
|
||||
with open(self.matrix_file, 'r') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def check_services(self) -> bool:
|
||||
"""Check if FuzzForge services are running."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "--filter", "name=fuzzforge-backend", "--format", "{{.Status}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
return "Up" in result.stdout
|
||||
except Exception as e:
|
||||
print(f"❌ Error checking services: {e}")
|
||||
return False
|
||||
|
||||
def start_services(self) -> bool:
|
||||
"""Start FuzzForge services if not running."""
|
||||
if self.check_services():
|
||||
print("✅ FuzzForge services already running")
|
||||
return True
|
||||
|
||||
print("🚀 Starting FuzzForge services...")
|
||||
try:
|
||||
subprocess.run(
|
||||
["docker", "compose", "up", "-d"],
|
||||
cwd=self.root_dir,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
# Wait for services to be ready
|
||||
print("⏳ Waiting for services to be ready...")
|
||||
max_wait = 60
|
||||
waited = 0
|
||||
while waited < max_wait:
|
||||
if self.check_services():
|
||||
print("✅ Services ready")
|
||||
time.sleep(5) # Extra wait for full initialization
|
||||
return True
|
||||
time.sleep(2)
|
||||
waited += 2
|
||||
print(f"⚠️ Services did not become ready within {max_wait}s")
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"❌ Failed to start services: {e}")
|
||||
return False
|
||||
|
||||
def execute_workflow(
|
||||
self,
|
||||
workflow_name: str,
|
||||
config: Dict[str, Any],
|
||||
platform: Optional[str] = None
|
||||
) -> WorkflowTestResult:
|
||||
"""Execute a single workflow and validate results."""
|
||||
start_time = time.time()
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Testing workflow: {workflow_name}")
|
||||
if platform:
|
||||
print(f"Platform: {platform}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Build command
|
||||
working_dir = self.root_dir / config['working_directory']
|
||||
cmd = [
|
||||
"ff", "workflow", "run",
|
||||
workflow_name,
|
||||
".",
|
||||
"--wait",
|
||||
"--no-interactive"
|
||||
]
|
||||
|
||||
# Add parameters
|
||||
params = config.get('parameters', {})
|
||||
for key, value in params.items():
|
||||
if isinstance(value, (str, int, float)):
|
||||
cmd.append(f"{key}={value}")
|
||||
|
||||
# Add SARIF export if expected
|
||||
sarif_file = None
|
||||
if config.get('expected', {}).get('sarif_export'):
|
||||
sarif_file = working_dir / f"test-{workflow_name}.sarif"
|
||||
cmd.extend(["--export-sarif", str(sarif_file)])
|
||||
|
||||
# Execute workflow
|
||||
print(f"Command: {' '.join(cmd)}")
|
||||
print(f"Working directory: {working_dir}")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=working_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=config.get('timeout', 300)
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
print(f"\n⏱️ Duration: {duration:.2f}s")
|
||||
|
||||
# Parse output for run_id
|
||||
run_id = self._extract_run_id(result.stdout)
|
||||
|
||||
# Check if workflow completed successfully
|
||||
if result.returncode != 0:
|
||||
error_msg = result.stderr or result.stdout
|
||||
print(f"❌ Workflow failed with exit code {result.returncode}")
|
||||
print(f"Error: {error_msg[:500]}")
|
||||
return WorkflowTestResult(
|
||||
workflow_name=workflow_name,
|
||||
success=False,
|
||||
duration=duration,
|
||||
run_id=run_id,
|
||||
error=error_msg[:500]
|
||||
)
|
||||
|
||||
# Validate SARIF export
|
||||
sarif_exported = False
|
||||
if sarif_file and sarif_file.exists():
|
||||
sarif_exported = self._validate_sarif(sarif_file)
|
||||
print(f"✅ SARIF export validated" if sarif_exported else "⚠️ SARIF export invalid")
|
||||
|
||||
# Get findings count
|
||||
findings_count = self._count_findings(run_id) if run_id else None
|
||||
|
||||
print(f"✅ Workflow completed successfully")
|
||||
if findings_count is not None:
|
||||
print(f" Findings: {findings_count}")
|
||||
|
||||
return WorkflowTestResult(
|
||||
workflow_name=workflow_name,
|
||||
success=True,
|
||||
duration=duration,
|
||||
status="COMPLETED",
|
||||
run_id=run_id,
|
||||
findings_count=findings_count,
|
||||
sarif_exported=sarif_exported
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
duration = time.time() - start_time
|
||||
print(f"❌ Workflow timed out after {duration:.2f}s")
|
||||
return WorkflowTestResult(
|
||||
workflow_name=workflow_name,
|
||||
success=False,
|
||||
duration=duration,
|
||||
error=f"Timeout after {config.get('timeout')}s"
|
||||
)
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
print(f"❌ Unexpected error: {e}")
|
||||
return WorkflowTestResult(
|
||||
workflow_name=workflow_name,
|
||||
success=False,
|
||||
duration=duration,
|
||||
error=str(e)
|
||||
)
|
||||
|
||||
def _extract_run_id(self, output: str) -> Optional[str]:
|
||||
"""Extract run_id from workflow output."""
|
||||
for line in output.split('\n'):
|
||||
if 'run_id' in line.lower() or 'execution id' in line.lower():
|
||||
# Try to extract the ID
|
||||
parts = line.split()
|
||||
for part in parts:
|
||||
if '-' in part and len(part) > 10:
|
||||
return part.strip(',:')
|
||||
return None
|
||||
|
||||
def _validate_sarif(self, sarif_file: Path) -> bool:
|
||||
"""Validate SARIF file structure."""
|
||||
try:
|
||||
with open(sarif_file, 'r') as f:
|
||||
sarif = json.load(f)
|
||||
# Basic SARIF validation
|
||||
return (
|
||||
'version' in sarif and
|
||||
'runs' in sarif and
|
||||
isinstance(sarif['runs'], list)
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"⚠️ SARIF validation error: {e}")
|
||||
return False
|
||||
|
||||
def _count_findings(self, run_id: str) -> Optional[int]:
|
||||
"""Count findings for a run."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ff", "findings", "list", run_id, "--format", "json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
if result.returncode == 0:
|
||||
findings = json.loads(result.stdout)
|
||||
return len(findings) if isinstance(findings, list) else 0
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def run_suite(self, suite_name: str) -> bool:
|
||||
"""Run a predefined test suite."""
|
||||
suite = self.matrix.get('test_suites', {}).get(suite_name)
|
||||
if not suite:
|
||||
print(f"❌ Suite '{suite_name}' not found")
|
||||
return False
|
||||
|
||||
workflows = suite.get('workflows', [])
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Running test suite: {suite_name}")
|
||||
print(f"Workflows: {', '.join(workflows)}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
for workflow_name in workflows:
|
||||
config = self.matrix['workflows'].get(workflow_name)
|
||||
if not config:
|
||||
print(f"⚠️ Workflow '{workflow_name}' not found in matrix")
|
||||
continue
|
||||
|
||||
result = self.execute_workflow(workflow_name, config)
|
||||
self.results.append(result)
|
||||
|
||||
return self.print_summary()
|
||||
|
||||
def run_workflow(self, workflow_name: str, platform: Optional[str] = None) -> bool:
|
||||
"""Run a single workflow."""
|
||||
config = self.matrix['workflows'].get(workflow_name)
|
||||
if not config:
|
||||
print(f"❌ Workflow '{workflow_name}' not found")
|
||||
return False
|
||||
|
||||
result = self.execute_workflow(workflow_name, config, platform)
|
||||
self.results.append(result)
|
||||
|
||||
return result.success
|
||||
|
||||
def print_summary(self) -> bool:
|
||||
"""Print test summary."""
|
||||
print(f"\n\n{'='*60}")
|
||||
print("TEST SUMMARY")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
total = len(self.results)
|
||||
passed = sum(1 for r in self.results if r.success)
|
||||
failed = total - passed
|
||||
|
||||
print(f"Total tests: {total}")
|
||||
print(f"Passed: {passed} ✅")
|
||||
print(f"Failed: {failed} ❌")
|
||||
print()
|
||||
|
||||
if failed > 0:
|
||||
print("Failed tests:")
|
||||
for result in self.results:
|
||||
if not result.success:
|
||||
print(f" - {result.workflow_name}")
|
||||
if result.error:
|
||||
print(f" Error: {result.error[:100]}")
|
||||
|
||||
print(f"\n{'='*60}\n")
|
||||
return failed == 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Test FuzzForge workflows")
|
||||
parser.add_argument(
|
||||
"--suite",
|
||||
choices=["fast", "full", "platform"],
|
||||
help="Test suite to run"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workflow",
|
||||
help="Single workflow to test"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
help="Platform for platform-specific testing (e.g., linux/amd64)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--matrix",
|
||||
type=Path,
|
||||
default=Path(".github/test-matrix.yaml"),
|
||||
help="Path to test matrix file"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-service-start",
|
||||
action="store_true",
|
||||
help="Skip starting services (assume already running)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine root directory
|
||||
root_dir = Path(__file__).parent.parent
|
||||
|
||||
# Load tester
|
||||
matrix_file = root_dir / args.matrix
|
||||
if not matrix_file.exists():
|
||||
print(f"❌ Matrix file not found: {matrix_file}")
|
||||
sys.exit(1)
|
||||
|
||||
tester = WorkflowTester(matrix_file, root_dir)
|
||||
|
||||
# Start services if needed
|
||||
if not args.skip_service_start:
|
||||
if not tester.start_services():
|
||||
print("❌ Failed to start services")
|
||||
sys.exit(1)
|
||||
|
||||
# Run tests
|
||||
success = False
|
||||
if args.suite:
|
||||
success = tester.run_suite(args.suite)
|
||||
elif args.workflow:
|
||||
success = tester.run_workflow(args.workflow, args.platform)
|
||||
else:
|
||||
print("❌ Must specify --suite or --workflow")
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
17
volumes/env/.env.example
vendored
17
volumes/env/.env.example
vendored
@@ -1,17 +0,0 @@
|
||||
# FuzzForge Agent Configuration
|
||||
# Copy this to .env and configure your API keys
|
||||
|
||||
# LiteLLM Model Configuration
|
||||
LITELLM_MODEL=gemini/gemini-2.0-flash-001
|
||||
# LITELLM_PROVIDER=gemini
|
||||
|
||||
# API Keys (uncomment and configure as needed)
|
||||
# GOOGLE_API_KEY=
|
||||
# OPENAI_API_KEY=
|
||||
# ANTHROPIC_API_KEY=
|
||||
# OPENROUTER_API_KEY=
|
||||
# MISTRAL_API_KEY=
|
||||
|
||||
# Agent Configuration
|
||||
# DEFAULT_TIMEOUT=120
|
||||
# DEFAULT_CONTEXT_ID=default
|
||||
65
volumes/env/.env.template
vendored
Normal file
65
volumes/env/.env.template
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
# =============================================================================
|
||||
# FuzzForge LiteLLM Proxy Configuration
|
||||
# =============================================================================
|
||||
# Copy this file to .env and fill in your API keys
|
||||
# Bootstrap will automatically create virtual keys for each service
|
||||
# =============================================================================
|
||||
|
||||
# LiteLLM Proxy Internal Configuration
|
||||
# -----------------------------------------------------------------------------
|
||||
FF_LLM_PROXY_BASE_URL=http://llm-proxy:4000
|
||||
LITELLM_MASTER_KEY=sk-master-test
|
||||
LITELLM_SALT_KEY=super-secret-salt
|
||||
|
||||
# Default Models (comma-separated, leave empty for model-agnostic access)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Examples:
|
||||
# openai/gpt-5-mini,openai/text-embedding-3-large
|
||||
# anthropic/claude-sonnet-4-5-20250929,openai/gpt-5-mini
|
||||
# (empty = unrestricted access to all registered models)
|
||||
LITELLM_DEFAULT_MODELS=
|
||||
|
||||
# Upstream Provider API Keys
|
||||
# -----------------------------------------------------------------------------
|
||||
# Add your real provider keys here - these are used by the proxy to call LLM providers
|
||||
LITELLM_OPENAI_API_KEY=your-openai-key-here
|
||||
LITELLM_ANTHROPIC_API_KEY=your-anthropic-key-here
|
||||
LITELLM_GEMINI_API_KEY=
|
||||
LITELLM_MISTRAL_API_KEY=
|
||||
LITELLM_OPENROUTER_API_KEY=
|
||||
|
||||
# Virtual Keys Budget & Duration Configuration
|
||||
# -----------------------------------------------------------------------------
|
||||
# These control the budget and duration for auto-generated virtual keys
|
||||
# Task Agent Key - used by task-agent service for A2A LiteLLM calls
|
||||
TASK_AGENT_BUDGET=25.0
|
||||
TASK_AGENT_DURATION=30d
|
||||
|
||||
# Cognee Key - used by Cognee for knowledge graph ingestion and queries
|
||||
COGNEE_BUDGET=50.0
|
||||
COGNEE_DURATION=30d
|
||||
|
||||
# General CLI/SDK Key - used by ff CLI and fuzzforge-sdk
|
||||
CLI_BUDGET=100.0
|
||||
CLI_DURATION=30d
|
||||
|
||||
# Virtual Keys (auto-generated by bootstrap - leave blank)
|
||||
# -----------------------------------------------------------------------------
|
||||
TASK_AGENT_API_KEY=
|
||||
COGNEE_API_KEY=
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# LiteLLM Proxy Client Configuration
|
||||
# -----------------------------------------------------------------------------
|
||||
# For CLI and SDK usage (Cognee, ff ingest, etc.)
|
||||
LITELLM_PROXY_API_BASE=http://localhost:10999
|
||||
LLM_ENDPOINT=http://localhost:10999
|
||||
LLM_PROVIDER=openai
|
||||
LLM_MODEL=litellm_proxy/gpt-5-mini
|
||||
LLM_API_BASE=http://localhost:10999
|
||||
LLM_EMBEDDING_MODEL=litellm_proxy/text-embedding-3-large
|
||||
|
||||
# UI Access
|
||||
# -----------------------------------------------------------------------------
|
||||
UI_USERNAME=fuzzforge
|
||||
UI_PASSWORD=fuzzforge123
|
||||
95
volumes/env/README.md
vendored
95
volumes/env/README.md
vendored
@@ -1,22 +1,89 @@
|
||||
# FuzzForge Environment Configuration
|
||||
# FuzzForge LiteLLM Proxy Configuration
|
||||
|
||||
This directory contains environment files that are mounted into Docker containers.
|
||||
This directory contains configuration for the LiteLLM proxy with model-agnostic virtual keys.
|
||||
|
||||
## Quick Start (Fresh Clone)
|
||||
|
||||
### 1. Create Your `.env` File
|
||||
|
||||
```bash
|
||||
cp .env.template .env
|
||||
```
|
||||
|
||||
### 2. Add Your Provider API Keys
|
||||
|
||||
Edit `.env` and add your **real** API keys:
|
||||
|
||||
```bash
|
||||
LITELLM_OPENAI_API_KEY=sk-proj-YOUR-OPENAI-KEY-HERE
|
||||
LITELLM_ANTHROPIC_API_KEY=sk-ant-api03-YOUR-ANTHROPIC-KEY-HERE
|
||||
```
|
||||
|
||||
### 3. Start Services
|
||||
|
||||
```bash
|
||||
cd ../.. # Back to repo root
|
||||
COMPOSE_PROFILES=secrets docker compose up -d
|
||||
```
|
||||
|
||||
Bootstrap will automatically:
|
||||
- Generate 3 virtual keys with individual budgets
|
||||
- Write them to your `.env` file
|
||||
- No model restrictions (model-agnostic)
|
||||
|
||||
## Files
|
||||
|
||||
- `.env.example` - Template configuration file
|
||||
- `.env` - Your actual configuration (create by copying .env.example)
|
||||
- **`.env.template`** - Clean template (checked into git)
|
||||
- **`.env`** - Your real keys (git ignored, you create this)
|
||||
- **`.env.example`** - Legacy example
|
||||
|
||||
## Usage
|
||||
## Virtual Keys (Auto-Generated)
|
||||
|
||||
1. Copy the example file:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
Bootstrap creates 3 keys with budget controls:
|
||||
|
||||
2. Edit `.env` and add your API keys
|
||||
| Key | Budget | Duration | Used By |
|
||||
|-----|--------|----------|---------|
|
||||
| `OPENAI_API_KEY` | $100 | 30 days | CLI, SDK |
|
||||
| `TASK_AGENT_API_KEY` | $25 | 30 days | Task Agent |
|
||||
| `COGNEE_API_KEY` | $50 | 30 days | Cognee |
|
||||
|
||||
3. Restart Docker containers to apply changes:
|
||||
```bash
|
||||
docker-compose restart
|
||||
```
|
||||
All keys are **model-agnostic** by default (no restrictions).
|
||||
|
||||
## Using Models
|
||||
|
||||
Registered models in `volumes/litellm/proxy_config.yaml`:
|
||||
- `gpt-5-mini` → `openai/gpt-5-mini`
|
||||
- `claude-sonnet-4-5` → `anthropic/claude-sonnet-4-5-20250929`
|
||||
- `text-embedding-3-large` → `openai/text-embedding-3-large`
|
||||
|
||||
### Use Registered Aliases:
|
||||
|
||||
```bash
|
||||
fuzzforge workflow run llm_secret_detection . -n llm_model=gpt-5-mini
|
||||
fuzzforge workflow run llm_secret_detection . -n llm_model=claude-sonnet-4-5
|
||||
```
|
||||
|
||||
### Use Any Model (Direct):
|
||||
|
||||
```bash
|
||||
# Works without registering first!
|
||||
fuzzforge workflow run llm_secret_detection . -n llm_model=openai/gpt-5-nano
|
||||
```
|
||||
|
||||
## Proxy UI
|
||||
|
||||
http://localhost:10999/ui
|
||||
- User: `fuzzforge` / Pass: `fuzzforge123`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
```bash
|
||||
# Check bootstrap logs
|
||||
docker compose logs llm-proxy-bootstrap
|
||||
|
||||
# Verify keys generated
|
||||
grep "API_KEY=" .env | grep -v "^#" | grep -v "your-"
|
||||
|
||||
# Restart services
|
||||
docker compose restart llm-proxy task-agent
|
||||
```
|
||||
|
||||
26
volumes/litellm/proxy_config.yaml
Normal file
26
volumes/litellm/proxy_config.yaml
Normal file
@@ -0,0 +1,26 @@
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
store_model_in_db: true
|
||||
store_prompts_in_spend_logs: true
|
||||
otel: true
|
||||
|
||||
litellm_settings:
|
||||
callbacks:
|
||||
- "otel"
|
||||
|
||||
model_list:
|
||||
- model_name: claude-sonnet-4-5
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5-20250929
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
- model_name: gpt-5-mini
|
||||
litellm_params:
|
||||
model: openai/gpt-5-mini
|
||||
api_key: os.environ/LITELLM_OPENAI_API_KEY
|
||||
|
||||
- model_name: text-embedding-3-large
|
||||
litellm_params:
|
||||
model: openai/text-embedding-3-large
|
||||
api_key: os.environ/LITELLM_OPENAI_API_KEY
|
||||
25
volumes/otel/collector-config.yaml
Normal file
25
volumes/otel/collector-config.yaml
Normal file
@@ -0,0 +1,25 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
batch:
|
||||
|
||||
exporters:
|
||||
debug:
|
||||
verbosity: detailed
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [debug]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [debug]
|
||||
Reference in New Issue
Block a user