mirror of
https://github.com/FuzzingLabs/fuzzforge_ai.git
synced 2026-02-25 15:05:55 +00:00
feat(test): add automated workflow testing framework
- Add test matrix configuration (.github/test-matrix.yaml) - Maps 8 workflows to workers, test projects, and parameters - Excludes LLM and OSS-Fuzz workflows - Defines fast, full, and platform test suites - Add workflow execution test script (scripts/test_workflows.py) - Executes workflows with parameter validation - Validates SARIF export and structure - Counts findings and measures execution time - Generates test summary reports - Add platform detection unit tests (cli/tests/test_platform_detection.py) - Tests platform detection (x86_64, aarch64, arm64) - Tests Dockerfile selection for multi-platform workers - Tests metadata.yaml parsing - Includes integration tests - Add GitHub Actions workflow (.github/workflows/test-workflows.yml) - Platform detection unit tests - Fast workflow tests (5 workflows on every PR) - Android platform-specific tests (AMD64 + ARM64) - Full workflow tests (on main/schedule) - Automatic log collection on failure - Add comprehensive testing documentation (docs/docs/development/testing.md) - Local testing guide - CI/CD testing explanation - Platform-specific testing guide - Debugging guide and best practices - Update test.yml with reference to new workflow tests - Remove tracked .fuzzforge/findings.db (already in .gitignore) Tested locally: - Single workflow test: python_sast (6.87s) ✅ - Fast test suite: 5/5 workflows passed ✅ - android_static_analysis (98.98s) ✅ - python_sast (6.78s) ✅ - secret_detection (38.04s) ✅ - gitleaks_detection (1.67s) ✅ - trufflehog_detection (1.64s) ✅
This commit is contained in:
tduhamel42
178
.github/test-matrix.yaml
vendored
Normal file
178
.github/test-matrix.yaml
vendored
Normal file
@@ -0,0 +1,178 @@
|
||||
# Test Matrix Configuration for Automated Workflow Testing
|
||||
#
|
||||
# This file defines which workflows to test, their required workers,
|
||||
# test projects, parameters, and expected outcomes.
|
||||
#
|
||||
# Excluded workflows:
|
||||
# - llm_analysis (requires LLM API keys)
|
||||
# - llm_secret_detection (requires LLM API keys)
|
||||
# - ossfuzz_campaign (requires OSS-Fuzz project configuration)
|
||||
|
||||
version: "1.0"
|
||||
|
||||
# Worker to Dockerfile mapping
|
||||
workers:
|
||||
android:
|
||||
dockerfiles:
|
||||
linux/amd64: "Dockerfile.amd64"
|
||||
linux/arm64: "Dockerfile.arm64"
|
||||
metadata: "workers/android/metadata.yaml"
|
||||
|
||||
python:
|
||||
dockerfiles:
|
||||
default: "Dockerfile"
|
||||
|
||||
rust:
|
||||
dockerfiles:
|
||||
default: "Dockerfile"
|
||||
|
||||
secrets:
|
||||
dockerfiles:
|
||||
default: "Dockerfile"
|
||||
|
||||
# Workflow test configurations
|
||||
workflows:
|
||||
# Android Static Analysis
|
||||
android_static_analysis:
|
||||
worker: android
|
||||
test_project: test_projects/android_test
|
||||
working_directory: test_projects/android_test
|
||||
parameters:
|
||||
apk_path: "BeetleBug.apk"
|
||||
timeout: 300
|
||||
platform_specific: true # Test on both amd64 and arm64
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [android, static-analysis, fast]
|
||||
|
||||
# Python SAST
|
||||
python_sast:
|
||||
worker: python
|
||||
test_project: test_projects/vulnerable_app
|
||||
working_directory: test_projects/vulnerable_app
|
||||
parameters: {}
|
||||
timeout: 180
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [python, sast, fast]
|
||||
|
||||
# Python Fuzzing (Atheris)
|
||||
atheris_fuzzing:
|
||||
worker: python
|
||||
test_project: test_projects/python_fuzz_waterfall
|
||||
working_directory: test_projects/python_fuzz_waterfall
|
||||
parameters:
|
||||
max_total_time: 30 # Short fuzzing run for testing
|
||||
artifact_prefix: "test-atheris"
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: false # May not find crashes in short run
|
||||
sarif_export: false
|
||||
tags: [python, fuzzing, slow]
|
||||
|
||||
# Rust Fuzzing (cargo-fuzz)
|
||||
cargo_fuzzing:
|
||||
worker: rust
|
||||
test_project: test_projects/rust_fuzz_test
|
||||
working_directory: test_projects/rust_fuzz_test
|
||||
parameters:
|
||||
max_total_time: 30 # Short fuzzing run for testing
|
||||
artifact_prefix: "test-cargo"
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: false # May not find crashes in short run
|
||||
sarif_export: false
|
||||
tags: [rust, fuzzing, slow]
|
||||
|
||||
# Secret Detection (combined)
|
||||
secret_detection:
|
||||
worker: secrets
|
||||
test_project: test_projects/secret_detection_benchmark
|
||||
working_directory: test_projects/secret_detection_benchmark
|
||||
parameters: {}
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [secrets, detection, fast]
|
||||
|
||||
# Gitleaks Detection
|
||||
gitleaks_detection:
|
||||
worker: secrets
|
||||
test_project: test_projects/secret_detection_benchmark
|
||||
working_directory: test_projects/secret_detection_benchmark
|
||||
parameters: {}
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [secrets, gitleaks, fast]
|
||||
|
||||
# TruffleHog Detection
|
||||
trufflehog_detection:
|
||||
worker: secrets
|
||||
test_project: test_projects/secret_detection_benchmark
|
||||
working_directory: test_projects/secret_detection_benchmark
|
||||
parameters: {}
|
||||
timeout: 120
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [secrets, trufflehog, fast]
|
||||
|
||||
# Security Assessment (composite workflow)
|
||||
security_assessment:
|
||||
worker: python # Uses multiple workers internally
|
||||
test_project: test_projects/vulnerable_app
|
||||
working_directory: test_projects/vulnerable_app
|
||||
parameters: {}
|
||||
timeout: 300
|
||||
expected:
|
||||
status: "COMPLETED"
|
||||
has_findings: true
|
||||
sarif_export: true
|
||||
tags: [composite, security, slow]
|
||||
|
||||
# Test suites - groups of workflows for different scenarios
|
||||
test_suites:
|
||||
# Fast tests - run on every PR
|
||||
fast:
|
||||
workflows:
|
||||
- android_static_analysis
|
||||
- python_sast
|
||||
- secret_detection
|
||||
- gitleaks_detection
|
||||
- trufflehog_detection
|
||||
timeout: 900 # 15 minutes total
|
||||
|
||||
# Full tests - run on main/master
|
||||
full:
|
||||
workflows:
|
||||
- android_static_analysis
|
||||
- python_sast
|
||||
- atheris_fuzzing
|
||||
- cargo_fuzzing
|
||||
- secret_detection
|
||||
- gitleaks_detection
|
||||
- trufflehog_detection
|
||||
- security_assessment
|
||||
timeout: 1800 # 30 minutes total
|
||||
|
||||
# Platform-specific tests - test Dockerfile selection
|
||||
platform:
|
||||
workflows:
|
||||
- android_static_analysis
|
||||
- python_sast
|
||||
platforms:
|
||||
- linux/amd64
|
||||
- linux/arm64
|
||||
timeout: 600 # 10 minutes total
|
||||
319
.github/workflows/test-workflows.yml
vendored
Normal file
319
.github/workflows/test-workflows.yml
vendored
Normal file
@@ -0,0 +1,319 @@
|
||||
name: Workflow Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, dev, develop, test/** ]
|
||||
pull_request:
|
||||
branches: [ main, master, dev, develop ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_suite:
|
||||
description: 'Test suite to run'
|
||||
required: false
|
||||
default: 'fast'
|
||||
type: choice
|
||||
options:
|
||||
- fast
|
||||
- full
|
||||
- platform
|
||||
|
||||
jobs:
|
||||
#############################################################################
|
||||
# Platform Detection Unit Tests
|
||||
#############################################################################
|
||||
platform-detection-tests:
|
||||
name: Platform Detection Unit Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pytest pytest-cov pyyaml
|
||||
pip install -e .
|
||||
|
||||
- name: Run platform detection tests
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
pytest tests/test_platform_detection.py -v \
|
||||
--cov=src/fuzzforge_cli \
|
||||
--cov-report=term \
|
||||
--cov-report=xml
|
||||
|
||||
- name: Upload coverage
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
file: ./cli/coverage.xml
|
||||
flags: cli-platform-detection
|
||||
name: cli-platform-detection
|
||||
|
||||
#############################################################################
|
||||
# Fast Workflow Tests (AMD64 only)
|
||||
#############################################################################
|
||||
fast-workflow-tests:
|
||||
name: Fast Workflow Tests (AMD64)
|
||||
runs-on: ubuntu-latest
|
||||
needs: platform-detection-tests
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install FuzzForge CLI
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e .
|
||||
pip install pyyaml # Required by test script
|
||||
|
||||
- name: Copy environment template
|
||||
run: |
|
||||
mkdir -p volumes/env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
- name: Start FuzzForge services
|
||||
run: |
|
||||
docker compose up -d
|
||||
echo "⏳ Waiting for services to be ready..."
|
||||
sleep 30
|
||||
|
||||
# Wait for backend to be healthy
|
||||
max_wait=60
|
||||
waited=0
|
||||
while [ $waited -lt $max_wait ]; do
|
||||
if docker ps --filter "name=fuzzforge-backend" --format "{{.Status}}" | grep -q "healthy"; then
|
||||
echo "✅ Backend is healthy"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for backend... ($waited/$max_wait seconds)"
|
||||
sleep 5
|
||||
waited=$((waited + 5))
|
||||
done
|
||||
|
||||
- name: Run fast workflow tests
|
||||
run: |
|
||||
python scripts/test_workflows.py --suite fast --skip-service-start
|
||||
timeout-minutes: 20
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Docker container status ==="
|
||||
docker ps -a
|
||||
|
||||
echo "=== Backend logs ==="
|
||||
docker logs fuzzforge-backend --tail 100
|
||||
|
||||
echo "=== Worker logs ==="
|
||||
for worker in python secrets android; do
|
||||
if docker ps -a --format "{{.Names}}" | grep -q "fuzzforge-worker-$worker"; then
|
||||
echo "=== Worker: $worker ==="
|
||||
docker logs fuzzforge-worker-$worker --tail 50
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
#############################################################################
|
||||
# Platform-Specific Tests (Android Worker)
|
||||
#############################################################################
|
||||
android-platform-tests:
|
||||
name: Android Worker Platform Tests
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs: platform-detection-tests
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
platform: linux/amd64
|
||||
arch: x86_64
|
||||
# ARM64 runner (uncomment when GitHub Actions ARM64 runners are available)
|
||||
# - os: ubuntu-24.04-arm
|
||||
# platform: linux/arm64
|
||||
# arch: aarch64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install FuzzForge CLI
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e .
|
||||
pip install pyyaml
|
||||
|
||||
- name: Verify platform detection
|
||||
run: |
|
||||
echo "Expected platform: ${{ matrix.platform }}"
|
||||
echo "Expected arch: ${{ matrix.arch }}"
|
||||
echo "Actual arch: $(uname -m)"
|
||||
|
||||
# Verify platform matches
|
||||
if [ "$(uname -m)" != "${{ matrix.arch }}" ]; then
|
||||
echo "❌ Platform mismatch!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Check Android worker Dockerfile selection
|
||||
run: |
|
||||
# Check which Dockerfile would be selected
|
||||
if [ "${{ matrix.platform }}" == "linux/amd64" ]; then
|
||||
expected_dockerfile="Dockerfile.amd64"
|
||||
else
|
||||
expected_dockerfile="Dockerfile.arm64"
|
||||
fi
|
||||
|
||||
echo "Expected Dockerfile: $expected_dockerfile"
|
||||
|
||||
# Verify the Dockerfile exists
|
||||
if [ ! -f "workers/android/$expected_dockerfile" ]; then
|
||||
echo "❌ Dockerfile not found: workers/android/$expected_dockerfile"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Dockerfile exists: $expected_dockerfile"
|
||||
|
||||
- name: Build Android worker for platform
|
||||
run: |
|
||||
echo "Building Android worker for platform: ${{ matrix.platform }}"
|
||||
docker compose build worker-android
|
||||
timeout-minutes: 15
|
||||
|
||||
- name: Copy environment template
|
||||
run: |
|
||||
mkdir -p volumes/env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
- name: Start FuzzForge services
|
||||
run: |
|
||||
docker compose up -d
|
||||
sleep 30
|
||||
|
||||
- name: Run Android workflow test
|
||||
run: |
|
||||
python scripts/test_workflows.py \
|
||||
--workflow android_static_analysis \
|
||||
--platform ${{ matrix.platform }} \
|
||||
--skip-service-start
|
||||
timeout-minutes: 10
|
||||
|
||||
- name: Verify correct Dockerfile was used
|
||||
run: |
|
||||
# Check docker image labels or inspect to verify correct build
|
||||
docker inspect fuzzforge-worker-android | grep -i "dockerfile" || true
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Android worker logs ==="
|
||||
docker logs fuzzforge-worker-android --tail 100
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
#############################################################################
|
||||
# Full Workflow Tests (on schedule or manual trigger)
|
||||
#############################################################################
|
||||
full-workflow-tests:
|
||||
name: Full Workflow Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: platform-detection-tests
|
||||
# Only run full tests on schedule, manual trigger, or main branch
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install FuzzForge CLI
|
||||
working-directory: ./cli
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e .
|
||||
pip install pyyaml
|
||||
|
||||
- name: Copy environment template
|
||||
run: |
|
||||
mkdir -p volumes/env
|
||||
cp volumes/env/.env.template volumes/env/.env
|
||||
|
||||
- name: Start FuzzForge services
|
||||
run: |
|
||||
docker compose up -d
|
||||
sleep 30
|
||||
|
||||
- name: Run full workflow tests
|
||||
run: |
|
||||
python scripts/test_workflows.py --suite full --skip-service-start
|
||||
timeout-minutes: 45
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Docker container status ==="
|
||||
docker ps -a
|
||||
|
||||
echo "=== All worker logs ==="
|
||||
for worker in python secrets rust android ossfuzz; do
|
||||
if docker ps -a --format "{{.Names}}" | grep -q "fuzzforge-worker-$worker"; then
|
||||
echo "=== Worker: $worker ==="
|
||||
docker logs fuzzforge-worker-$worker --tail 100
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
|
||||
#############################################################################
|
||||
# Test Summary
|
||||
#############################################################################
|
||||
test-summary:
|
||||
name: Workflow Test Summary
|
||||
runs-on: ubuntu-latest
|
||||
needs: [platform-detection-tests, fast-workflow-tests, android-platform-tests]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Check test results
|
||||
run: |
|
||||
if [ "${{ needs.platform-detection-tests.result }}" != "success" ]; then
|
||||
echo "❌ Platform detection tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${{ needs.fast-workflow-tests.result }}" != "success" ]; then
|
||||
echo "❌ Fast workflow tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${{ needs.android-platform-tests.result }}" != "success" ]; then
|
||||
echo "❌ Android platform tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ All workflow integration tests passed!"
|
||||
8
.github/workflows/test.yml
vendored
8
.github/workflows/test.yml
vendored
@@ -1,5 +1,13 @@
|
||||
name: Tests
|
||||
|
||||
# This workflow covers:
|
||||
# - Worker validation (Dockerfile and metadata checks)
|
||||
# - Docker image builds (only for modified workers)
|
||||
# - Python linting (ruff, mypy)
|
||||
# - Backend unit tests
|
||||
#
|
||||
# For end-to-end workflow integration tests, see: .github/workflows/test-workflows.yml
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master, dev, develop, feature/** ]
|
||||
|
||||
Reference in New Issue
Block a user