diff --git a/.github/ISSUE_TEMPLATE/workflow_submission.md b/.github/ISSUE_TEMPLATE/workflow_submission.md index 9ce4e25..92d692f 100644 --- a/.github/ISSUE_TEMPLATE/workflow_submission.md +++ b/.github/ISSUE_TEMPLATE/workflow_submission.md @@ -26,7 +26,7 @@ What type of security workflow is this? ## Files Please attach or provide links to your workflow files: -- [ ] `workflow.py` - Main Prefect flow implementation +- [ ] `workflow.py` - Main Temporal flow implementation - [ ] `Dockerfile` - Container definition - [ ] `metadata.yaml` - Workflow metadata - [ ] Test files or examples diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..a5b2a46 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,165 @@ +name: Benchmarks + +on: + # Disabled automatic runs - benchmarks not ready for CI/CD yet + # schedule: + # - cron: '0 2 * * *' # 2 AM UTC every day + + # Allow manual trigger for testing + workflow_dispatch: + inputs: + compare_with: + description: 'Baseline commit to compare against (optional)' + required: false + default: '' + + # pull_request: + # paths: + # - 'backend/benchmarks/**' + # - 'backend/toolbox/modules/**' + # - '.github/workflows/benchmark.yml' + +jobs: + benchmark: + name: Run Benchmarks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for comparison + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential + + - name: Install Python dependencies + working-directory: ./backend + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + pip install pytest pytest-asyncio pytest-benchmark pytest-benchmark[histogram] + pip install -e ../sdk # Install SDK for benchmarks + + - name: Run benchmarks + working-directory: ./backend + run: | + pytest benchmarks/ \ + -v \ + --benchmark-only \ + --benchmark-json=benchmark-results.json \ + --benchmark-histogram=benchmark-histogram + + - name: Store benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ github.run_number }} + path: | + backend/benchmark-results.json + backend/benchmark-histogram.svg + + - name: Download baseline benchmarks + if: github.event_name == 'pull_request' + uses: dawidd6/action-download-artifact@v3 + continue-on-error: true + with: + workflow: benchmark.yml + branch: ${{ github.base_ref }} + name: benchmark-results-* + path: ./baseline + search_artifacts: true + + - name: Compare with baseline + if: github.event_name == 'pull_request' && hashFiles('baseline/benchmark-results.json') != '' + run: | + python -c " + import json + import sys + + with open('backend/benchmark-results.json') as f: + current = json.load(f) + + with open('baseline/benchmark-results.json') as f: + baseline = json.load(f) + + print('\\n## Benchmark Comparison\\n') + print('| Benchmark | Current | Baseline | Change |') + print('|-----------|---------|----------|--------|') + + regressions = [] + + for bench in current['benchmarks']: + name = bench['name'] + current_time = bench['stats']['mean'] + + # Find matching baseline + baseline_bench = next((b for b in baseline['benchmarks'] if b['name'] == name), None) + if baseline_bench: + baseline_time = baseline_bench['stats']['mean'] + change = ((current_time - baseline_time) / baseline_time) * 100 + + print(f'| {name} | {current_time:.4f}s | {baseline_time:.4f}s | {change:+.2f}% |') + + # Flag regressions > 10% + if change > 10: + regressions.append((name, change)) + else: + print(f'| {name} | {current_time:.4f}s | N/A | NEW |') + + if regressions: + print('\\nāš ļø **Performance Regressions Detected:**') + for name, change in regressions: + print(f'- {name}: +{change:.2f}%') + sys.exit(1) + else: + print('\\nāœ… No significant performance regressions detected') + " + + - name: Comment PR with results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const results = JSON.parse(fs.readFileSync('backend/benchmark-results.json', 'utf8')); + + let body = '## Benchmark Results\\n\\n'; + body += '| Category | Benchmark | Mean Time | Std Dev |\\n'; + body += '|----------|-----------|-----------|---------|\\n'; + + for (const bench of results.benchmarks) { + const group = bench.group || 'ungrouped'; + const name = bench.name.split('::').pop(); + const mean = bench.stats.mean.toFixed(4); + const stddev = bench.stats.stddev.toFixed(4); + body += `| ${group} | ${name} | ${mean}s | ${stddev}s |\\n`; + } + + body += '\\nšŸ“Š Full benchmark results available in artifacts.'; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }); + + benchmark-summary: + name: Benchmark Summary + runs-on: ubuntu-latest + needs: benchmark + if: always() + steps: + - name: Check results + run: | + if [ "${{ needs.benchmark.result }}" != "success" ]; then + echo "Benchmarks failed or detected regressions" + exit 1 + fi + echo "Benchmarks completed successfully!" diff --git a/.github/workflows/examples/security-scan.yml b/.github/workflows/examples/security-scan.yml new file mode 100644 index 0000000..1fd4922 --- /dev/null +++ b/.github/workflows/examples/security-scan.yml @@ -0,0 +1,152 @@ +# FuzzForge CI/CD Example - Security Scanning +# +# This workflow demonstrates how to integrate FuzzForge into your CI/CD pipeline +# for automated security testing on pull requests and pushes. +# +# Features: +# - Runs entirely in GitHub Actions (no external infrastructure needed) +# - Auto-starts FuzzForge services on-demand +# - Fails builds on error-level SARIF findings +# - Uploads SARIF results to GitHub Security tab +# - Exports findings as artifacts +# +# Prerequisites: +# - Ubuntu runner with Docker support +# - At least 4GB RAM available +# - ~90 seconds startup time + +name: Security Scan Example + +on: + pull_request: + branches: [main, develop] + push: + branches: [main] + +jobs: + security-scan: + name: Security Assessment + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start FuzzForge + run: | + bash scripts/ci-start.sh + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install FuzzForge CLI + run: | + pip install ./cli + + - name: Initialize FuzzForge + run: | + ff init --api-url http://localhost:8000 --name "GitHub Actions Security Scan" + + - name: Run Security Assessment + run: | + ff workflow run security_assessment . \ + --wait \ + --fail-on error \ + --export-sarif results.sarif + + - name: Upload SARIF to GitHub Security + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif + + - name: Upload findings as artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: security-findings + path: results.sarif + retention-days: 30 + + - name: Stop FuzzForge + if: always() + run: | + bash scripts/ci-stop.sh + + secret-scan: + name: Secret Detection + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + + - name: Start FuzzForge + run: bash scripts/ci-start.sh + + - name: Install CLI + run: | + pip install ./cli + + - name: Initialize & Scan + run: | + ff init --api-url http://localhost:8000 --name "Secret Detection" + ff workflow run secret_detection . \ + --wait \ + --fail-on all \ + --export-sarif secrets.sarif + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: secret-scan-results + path: secrets.sarif + retention-days: 30 + + - name: Cleanup + if: always() + run: bash scripts/ci-stop.sh + + # Example: Nightly fuzzing campaign (long-running) + nightly-fuzzing: + name: Nightly Fuzzing + runs-on: ubuntu-latest + timeout-minutes: 120 + # Only run on schedule + if: github.event_name == 'schedule' + + steps: + - uses: actions/checkout@v4 + + - name: Start FuzzForge + run: bash scripts/ci-start.sh + + - name: Install CLI + run: pip install ./cli + + - name: Run Fuzzing Campaign + run: | + ff init --api-url http://localhost:8000 + ff workflow run atheris_fuzzing . \ + max_iterations=100000000 \ + timeout_seconds=7200 \ + --wait \ + --export-sarif fuzzing-results.sarif + # Don't fail on fuzzing findings, just report + continue-on-error: true + + - name: Upload fuzzing results + if: always() + uses: actions/upload-artifact@v4 + with: + name: fuzzing-results + path: fuzzing-results.sarif + retention-days: 90 + + - name: Cleanup + if: always() + run: bash scripts/ci-stop.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..03581ef --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,155 @@ +name: Tests + +on: + push: + branches: [ main, master, develop, feature/** ] + pull_request: + branches: [ main, master, develop ] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff mypy + + - name: Run ruff + run: ruff check backend/src backend/toolbox backend/tests backend/benchmarks --output-format=github + + - name: Run mypy (continue on error) + run: mypy backend/src backend/toolbox || true + continue-on-error: true + + unit-tests: + name: Unit Tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential + + - name: Install Python dependencies + working-directory: ./backend + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + pip install pytest pytest-asyncio pytest-cov pytest-xdist + + - name: Run unit tests + working-directory: ./backend + run: | + pytest tests/unit/ \ + -v \ + --cov=toolbox/modules \ + --cov=src \ + --cov-report=xml \ + --cov-report=term \ + --cov-report=html \ + -n auto + + - name: Upload coverage to Codecov + if: matrix.python-version == '3.11' + uses: codecov/codecov-action@v4 + with: + file: ./backend/coverage.xml + flags: unittests + name: codecov-backend + + - name: Upload coverage HTML + if: matrix.python-version == '3.11' + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: ./backend/htmlcov/ + + # integration-tests: + # name: Integration Tests + # runs-on: ubuntu-latest + # needs: unit-tests + # + # services: + # postgres: + # image: postgres:15 + # env: + # POSTGRES_USER: postgres + # POSTGRES_PASSWORD: postgres + # POSTGRES_DB: fuzzforge_test + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # ports: + # - 5432:5432 + # + # steps: + # - uses: actions/checkout@v4 + # + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # + # - name: Install Python dependencies + # working-directory: ./backend + # run: | + # python -m pip install --upgrade pip + # pip install -e ".[dev]" + # pip install pytest pytest-asyncio + # + # - name: Start services (Temporal, MinIO) + # run: | + # docker-compose -f docker-compose.yml up -d temporal minio + # sleep 30 + # + # - name: Run integration tests + # working-directory: ./backend + # run: | + # pytest tests/integration/ -v --tb=short + # env: + # DATABASE_URL: postgresql://postgres:postgres@localhost:5432/fuzzforge_test + # TEMPORAL_ADDRESS: localhost:7233 + # MINIO_ENDPOINT: localhost:9000 + # + # - name: Shutdown services + # if: always() + # run: docker-compose down + + test-summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [lint, unit-tests] + if: always() + steps: + - name: Check test results + run: | + if [ "${{ needs.unit-tests.result }}" != "success" ]; then + echo "Unit tests failed" + exit 1 + fi + echo "All tests passed!" diff --git a/.gitignore b/.gitignore index e6520c3..b090789 100644 --- a/.gitignore +++ b/.gitignore @@ -185,6 +185,9 @@ logs/ # FuzzForge project directories (user projects should manage their own .gitignore) .fuzzforge/ +# Docker volume configs (keep .env.example but ignore actual .env) +volumes/env/.env + # Test project databases and configurations test_projects/*/.fuzzforge/ test_projects/*/findings.db* @@ -201,6 +204,7 @@ dev_config.yaml reports/ output/ findings/ +*.sarif *.sarif.json *.html.report security_report.* @@ -229,6 +233,12 @@ yarn-error.log* *.key *.p12 *.pfx + +# Exception: Secret detection benchmark test files (not real secrets) +!test_projects/secret_detection_benchmark/ +!test_projects/secret_detection_benchmark/** +!**/secret_detection_benchmark_GROUND_TRUTH.json + secret* secrets/ credentials* @@ -288,4 +298,5 @@ test_projects/*/wallet.json test_projects/*/.npmrc test_projects/*/.git-credentials test_projects/*/credentials.* -test_projects/*/api_keys.* \ No newline at end of file +test_projects/*/api_keys.* +test_projects/*/ci-*.sh \ No newline at end of file diff --git a/.gitlab-ci.example.yml b/.gitlab-ci.example.yml new file mode 100644 index 0000000..57301ca --- /dev/null +++ b/.gitlab-ci.example.yml @@ -0,0 +1,121 @@ +# FuzzForge CI/CD Example - GitLab CI +# +# This file demonstrates how to integrate FuzzForge into your GitLab CI/CD pipeline. +# Copy this to `.gitlab-ci.yml` in your project root to enable security scanning. +# +# Features: +# - Runs entirely in GitLab runners (no external infrastructure) +# - Auto-starts FuzzForge services on-demand +# - Fails pipelines on critical/high severity findings +# - Uploads SARIF reports to GitLab Security Dashboard +# - Exports findings as artifacts +# +# Prerequisites: +# - GitLab Runner with Docker support (docker:dind) +# - At least 4GB RAM available +# - ~90 seconds startup time + +stages: + - security + +variables: + FUZZFORGE_API_URL: "http://localhost:8000" + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + +# Base template for all FuzzForge jobs +.fuzzforge_template: + image: docker:24 + services: + - docker:24-dind + before_script: + # Install dependencies + - apk add --no-cache bash curl python3 py3-pip git + # Start FuzzForge + - bash scripts/ci-start.sh + # Install CLI + - pip3 install ./cli --break-system-packages + # Initialize project + - ff init --api-url $FUZZFORGE_API_URL --name "GitLab CI Security Scan" + after_script: + # Cleanup + - bash scripts/ci-stop.sh || true + +# Security Assessment - Comprehensive code analysis +security:scan: + extends: .fuzzforge_template + stage: security + timeout: 30 minutes + script: + - ff workflow run security_assessment . --wait --fail-on error --export-sarif results.sarif + artifacts: + when: always + reports: + sast: results.sarif + paths: + - results.sarif + expire_in: 30 days + only: + - merge_requests + - main + - develop + +# Secret Detection - Scan for exposed credentials +security:secrets: + extends: .fuzzforge_template + stage: security + timeout: 15 minutes + script: + - ff workflow run secret_detection . --wait --fail-on all --export-sarif secrets.sarif + artifacts: + when: always + paths: + - secrets.sarif + expire_in: 30 days + only: + - merge_requests + - main + +# Nightly Fuzzing - Long-running fuzzing campaign (scheduled only) +security:fuzzing: + extends: .fuzzforge_template + stage: security + timeout: 2 hours + script: + - | + ff workflow run atheris_fuzzing . \ + max_iterations=100000000 \ + timeout_seconds=7200 \ + --wait \ + --export-sarif fuzzing-results.sarif + artifacts: + when: always + paths: + - fuzzing-results.sarif + expire_in: 90 days + allow_failure: true # Don't fail pipeline on fuzzing findings + only: + - schedules + +# OSS-Fuzz Campaign (for supported projects) +security:ossfuzz: + extends: .fuzzforge_template + stage: security + timeout: 1 hour + script: + - | + ff workflow run ossfuzz_campaign . \ + project_name=your-project-name \ + campaign_duration_hours=0.5 \ + --wait \ + --export-sarif ossfuzz-results.sarif + artifacts: + when: always + paths: + - ossfuzz-results.sarif + expire_in: 90 days + allow_failure: true + only: + - schedules + # Uncomment and set your project name + # when: manual diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..aa265b4 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,1020 @@ +# FuzzForge AI Architecture + +**Last Updated:** 2025-10-14 +**Status:** Production - Temporal with Vertical Workers + +--- + +## Table of Contents + +1. [Executive Summary](#executive-summary) +2. [Current Architecture (Temporal + Vertical Workers)](#current-architecture-temporal--vertical-workers) +3. [Vertical Worker Model](#vertical-worker-model) +4. [Storage Strategy (MinIO)](#storage-strategy-minio) +5. [Dynamic Workflow Loading](#dynamic-workflow-loading) +6. [Architecture Principles](#architecture-principles) +7. [Component Details](#component-details) +8. [Scaling Strategy](#scaling-strategy) +9. [File Lifecycle Management](#file-lifecycle-management) +10. [Future: Nomad Migration](#future-nomad-migration) + +--- + +## Executive Summary + +### The Architecture + +**Temporal orchestration** with a **vertical worker architecture** where each worker is pre-built with domain-specific security toolchains (Android, Rust, Web, iOS, Blockchain, OSS-Fuzz, etc.). Uses **MinIO** for unified S3-compatible storage across dev and production environments. + +### Key Architecture Features + +1. **Vertical Specialization:** Pre-built toolchains (Android: Frida, apktool; Rust: AFL++, cargo-fuzz) +2. **Zero Startup Overhead:** Long-lived workers (no container spawn per workflow) +3. **Dynamic Workflows:** Add workflows without rebuilding images (mount as volume) +4. **Unified Storage:** MinIO works identically in dev and prod +5. **Better Security:** No host filesystem mounts, isolated uploaded targets +6. **Automatic Cleanup:** MinIO lifecycle policies handle file expiration +7. **Scalability:** Clear path from single-host to multi-host to Nomad cluster + +--- + +## Current Architecture (Temporal + Vertical Workers) + +### Infrastructure Overview + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ FuzzForge Platform ā”‚ +ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ Temporal Server ā”‚ā—„ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”‚ MinIO (S3 Storage) ā”‚ ā”‚ +ā”‚ ā”‚ - Workflows ā”‚ ā”‚ - Uploaded targets ā”‚ ā”‚ +ā”‚ ā”‚ - State mgmt ā”‚ ā”‚ - Results (optional) ā”‚ ā”‚ +ā”‚ ā”‚ - Task queues ā”‚ ā”‚ - Lifecycle policies ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ +ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ (Task queue routing) ā”‚ +ā”‚ ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ Vertical Workers (Long-lived) ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ Android ā”‚ ā”‚ Rust/Native ā”‚ ā”‚ Web/JS ā”‚ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - apktool ā”‚ ā”‚ - AFL++ ā”‚ ā”‚ - Node.js ā”‚ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - Frida ā”‚ ā”‚ - cargo-fuzz ā”‚ ā”‚ - OWASP ZAP ā”‚ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - jadx ā”‚ ā”‚ - gdb ā”‚ ā”‚ - semgrep ā”‚ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - MobSF ā”‚ ā”‚ - valgrind ā”‚ ā”‚ - eslint ā”‚ā”‚ ā”‚ +ā”‚ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ iOS ā”‚ ā”‚ Blockchain ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - class-dump ā”‚ ā”‚ - mythril ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - Clutch ā”‚ ā”‚ - slither ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - Frida ā”‚ ā”‚ - echidna ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ - Hopper ā”‚ ā”‚ - manticore ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ ā”‚ +ā”‚ ā”‚ ā”‚ ā”‚ +ā”‚ ā”‚ All workers have: ā”‚ ā”‚ +ā”‚ ā”‚ - /app/toolbox mounted (workflow code) ā”‚ ā”‚ +ā”‚ ā”‚ - /cache for MinIO downloads ā”‚ ā”‚ +ā”‚ ā”‚ - Dynamic workflow discovery at startup ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ +``` + +### Service Breakdown + +```yaml +services: + temporal: # Workflow orchestration + embedded SQLite (dev) or Postgres (prod) + minio: # S3-compatible storage for targets and results + minio-setup: # One-time: create buckets, set policies + worker-android: # Android security vertical (scales independently) + worker-rust: # Rust/native security vertical + worker-web: # Web security vertical + # Additional verticals as needed: ios, blockchain, go, etc. + +Total: 6+ services (scales with verticals) +``` + +### Resource Usage + +``` +Temporal: ~500MB (includes embedded DB in dev) +MinIO: ~256MB (with CI_CD=true flag) +MinIO-setup: ~20MB (ephemeral, exits after setup) +Worker-android: ~512MB (varies by toolchain) +Worker-rust: ~512MB +Worker-web: ~512MB +ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +Total: ~2.3GB + +Note: +450MB overhead is worth it for: + - Unified dev/prod architecture + - No host filesystem mounts (security) + - Auto cleanup (lifecycle policies) + - Multi-host ready +``` + +--- + +## Vertical Worker Model + +### Concept + +Instead of generic workers that spawn workflow-specific containers, we have **specialized long-lived workers** pre-built with complete security toolchains for specific domains. + +### Vertical Taxonomy + +| Vertical | Tools Included | Use Cases | Workflows | +|----------|---------------|-----------|-----------| +| **android** | apktool, jadx, Frida, MobSF, androguard | APK analysis, reverse engineering, dynamic instrumentation | APK security assessment, malware analysis, repackaging detection | +| **rust** | AFL++, cargo-fuzz, gdb, valgrind, AddressSanitizer | Native fuzzing, memory safety | Cargo fuzzing campaigns, binary analysis | +| **web** | Node.js, OWASP ZAP, Burp Suite, semgrep, eslint | Web app security testing | XSS detection, SQL injection scanning, API fuzzing | +| **ios** | class-dump, Clutch, Frida, Hopper, ios-deploy | iOS app analysis | IPA analysis, jailbreak detection, runtime hooking | +| **blockchain** | mythril, slither, echidna, manticore, solc | Smart contract security | Solidity static analysis, property-based fuzzing | +| **go** | go-fuzz, staticcheck, gosec, dlv | Go security testing | Go fuzzing, static analysis | + +### Vertical Worker Architecture + +```dockerfile +# Example: workers/android/Dockerfile +FROM python:3.11-slim + +# Install Android SDK and tools +RUN apt-get update && apt-get install -y \ + openjdk-17-jdk \ + android-sdk \ + && rm -rf /var/lib/apt/lists/* + +# Install security tools +RUN pip install --no-cache-dir \ + apktool \ + androguard \ + frida-tools \ + pyaxmlparser + +# Install MobSF dependencies +RUN apt-get update && apt-get install -y \ + libxml2-dev \ + libxslt-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Temporal Python SDK +RUN pip install --no-cache-dir \ + temporalio \ + boto3 \ + pydantic + +# Copy worker entrypoint +COPY worker.py /app/ +WORKDIR /app + +# Worker will mount /app/toolbox and discover workflows at runtime +CMD ["python", "worker.py"] +``` + +### Dynamic Workflow Discovery + +```python +# workers/android/worker.py +import asyncio +from pathlib import Path +from temporalio.client import Client +from temporalio.worker import Worker + +async def discover_workflows(vertical: str): + """Discover workflows for this vertical from mounted toolbox""" + workflows = [] + toolbox = Path("/app/toolbox/workflows") + + for workflow_dir in toolbox.iterdir(): + if not workflow_dir.is_dir(): + continue + + metadata_file = workflow_dir / "metadata.yaml" + if not metadata_file.exists(): + continue + + # Parse metadata + with open(metadata_file) as f: + metadata = yaml.safe_load(f) + + # Check if workflow is for this vertical + if metadata.get("vertical") == vertical: + # Dynamically import workflow module + workflow_module = f"toolbox.workflows.{workflow_dir.name}.workflow" + module = __import__(workflow_module, fromlist=['']) + + # Find @workflow.defn decorated classes + for name, obj in inspect.getmembers(module, inspect.isclass): + if hasattr(obj, '__temporal_workflow_definition'): + workflows.append(obj) + logger.info(f"Discovered workflow: {name} for vertical {vertical}") + + return workflows + +async def main(): + vertical = os.getenv("WORKER_VERTICAL", "android") + temporal_address = os.getenv("TEMPORAL_ADDRESS", "localhost:7233") + + # Discover workflows for this vertical + workflows = await discover_workflows(vertical) + + if not workflows: + logger.warning(f"No workflows found for vertical: {vertical}") + return + + # Connect to Temporal + client = await Client.connect(temporal_address) + + # Start worker with discovered workflows + worker = Worker( + client, + task_queue=f"{vertical}-queue", + workflows=workflows, + activities=[ + get_target_activity, + cleanup_cache_activity, + # ... vertical-specific activities + ] + ) + + logger.info(f"Worker started for vertical '{vertical}' with {len(workflows)} workflows") + await worker.run() + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Workflow Declaration + +```yaml +# toolbox/workflows/android_apk_analysis/metadata.yaml +name: android_apk_analysis +version: 1.0.0 +description: "Deep analysis of Android APK files" +vertical: android # ā† Routes to worker-android +dependencies: + python: + - androguard==4.1.0 # Additional Python deps (optional) + - pyaxmlparser==0.3.28 +``` + +```python +# toolbox/workflows/android_apk_analysis/workflow.py +from temporalio import workflow +from datetime import timedelta + +@workflow.defn +class AndroidApkAnalysisWorkflow: + """ + Comprehensive Android APK security analysis + Runs in worker-android with apktool, Frida, jadx pre-installed + """ + + @workflow.run + async def run(self, target_id: str) -> dict: + # Activity 1: Download target from MinIO + apk_path = await workflow.execute_activity( + "get_target", + target_id, + start_to_close_timeout=timedelta(minutes=5) + ) + + # Activity 2: Extract manifest (uses apktool - pre-installed) + manifest = await workflow.execute_activity( + "extract_manifest", + apk_path, + start_to_close_timeout=timedelta(minutes=5) + ) + + # Activity 3: Static analysis (uses jadx - pre-installed) + static_results = await workflow.execute_activity( + "static_analysis", + apk_path, + start_to_close_timeout=timedelta(minutes=30) + ) + + # Activity 4: Frida instrumentation (uses Frida - pre-installed) + dynamic_results = await workflow.execute_activity( + "dynamic_analysis", + apk_path, + start_to_close_timeout=timedelta(hours=2) + ) + + # Activity 5: Cleanup local cache + await workflow.execute_activity( + "cleanup_cache", + apk_path, + start_to_close_timeout=timedelta(minutes=1) + ) + + return { + "manifest": manifest, + "static": static_results, + "dynamic": dynamic_results + } +``` + +--- + +## Storage Strategy (MinIO) + +### Why MinIO? + +**Goal:** Unified storage that works identically in dev and production, eliminating environment-specific code. + +**Alternatives considered:** +1. āŒ **LocalVolumeStorage** (mount /Users, /home): Security risk, platform-specific, doesn't scale +2. āŒ **Different storage per environment**: Complex, error-prone, dual maintenance +3. āœ… **MinIO everywhere**: Lightweight (+256MB), S3-compatible, multi-host ready + +### MinIO Configuration + +```yaml +# docker-compose.yaml +services: + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + ports: + - "9000:9000" # S3 API + - "9001:9001" # Web Console (http://localhost:9001) + volumes: + - minio_data:/data + environment: + MINIO_ROOT_USER: fuzzforge + MINIO_ROOT_PASSWORD: fuzzforge123 + MINIO_CI_CD: "true" # Reduces memory to 256MB (from 1GB) + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + + # One-time setup: create buckets and set lifecycle policies + minio-setup: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set fuzzforge http://minio:9000 fuzzforge fuzzforge123; + mc mb fuzzforge/targets --ignore-existing; + mc mb fuzzforge/results --ignore-existing; + mc ilm add fuzzforge/targets --expiry-days 7; + mc anonymous set download fuzzforge/results; + " +``` + +### Storage Backend Implementation + +```python +# backend/src/storage/s3_cached.py +import boto3 +from pathlib import Path +from datetime import datetime, timedelta +import logging + +logger = logging.getLogger(__name__) + +class S3CachedStorage: + """ + S3-compatible storage with local caching. + Works with MinIO (dev/prod) or AWS S3 (cloud). + """ + + def __init__(self): + self.s3 = boto3.client( + 's3', + endpoint_url=os.getenv('S3_ENDPOINT', 'http://minio:9000'), + aws_access_key_id=os.getenv('S3_ACCESS_KEY', 'fuzzforge'), + aws_secret_access_key=os.getenv('S3_SECRET_KEY', 'fuzzforge123') + ) + self.bucket = os.getenv('S3_BUCKET', 'targets') + self.cache_dir = Path(os.getenv('CACHE_DIR', '/cache')) + self.cache_max_size = self._parse_size(os.getenv('CACHE_MAX_SIZE', '10GB')) + self.cache_ttl = self._parse_duration(os.getenv('CACHE_TTL', '7d')) + + async def upload_target(self, file_path: Path, user_id: str) -> str: + """Upload target to MinIO and return target ID""" + target_id = str(uuid4()) + + # Upload with metadata for lifecycle management + self.s3.upload_file( + str(file_path), + self.bucket, + f'{target_id}/target', + ExtraArgs={ + 'Metadata': { + 'user_id': user_id, + 'uploaded_at': datetime.now().isoformat(), + 'filename': file_path.name + } + } + ) + + logger.info(f"Uploaded target {target_id} ({file_path.name})") + return target_id + + async def get_target(self, target_id: str) -> Path: + """ + Get target from cache or download from MinIO. + Returns local path to cached file. + """ + cache_path = self.cache_dir / target_id + cached_file = cache_path / "target" + + # Check cache + if cached_file.exists(): + # Update access time for LRU + cached_file.touch() + logger.info(f"Cache hit: {target_id}") + return cached_file + + # Cache miss - download from MinIO + logger.info(f"Cache miss: {target_id}, downloading from MinIO") + cache_path.mkdir(parents=True, exist_ok=True) + + self.s3.download_file( + self.bucket, + f'{target_id}/target', + str(cached_file) + ) + + return cached_file + + async def cleanup_cache(self): + """LRU eviction when cache exceeds max size""" + cache_files = [] + total_size = 0 + + for cache_file in self.cache_dir.rglob('*'): + if cache_file.is_file(): + stat = cache_file.stat() + cache_files.append({ + 'path': cache_file, + 'size': stat.st_size, + 'atime': stat.st_atime + }) + total_size += stat.st_size + + if total_size > self.cache_max_size: + # Sort by access time (oldest first) + cache_files.sort(key=lambda x: x['atime']) + + for file_info in cache_files: + if total_size <= self.cache_max_size: + break + + file_info['path'].unlink() + total_size -= file_info['size'] + logger.info(f"Evicted from cache: {file_info['path']}") +``` + +### Performance Characteristics + +| Operation | Direct Filesystem | MinIO (Local) | Impact | +|-----------|------------------|---------------|---------| +| Small file (<1MB) | ~1ms | ~5-10ms | Negligible for security workflows | +| Large file (>100MB) | ~200ms | ~220ms | ~10% overhead | +| Workflow duration | 5-60 minutes | 5-60 minutes + 2-4s upload | <1% overhead | +| Subsequent scans | Same | **Cached (0ms)** | Better than filesystem | + +**Verdict:** 2-4 second upload overhead is **negligible** for workflows that run 5-60 minutes. + +### Workspace Isolation + +To support concurrent workflows safely, FuzzForge implements workspace isolation with three modes: + +**1. Isolated Mode (Default)** +```python +# Each workflow run gets its own workspace +cache_path = f"/cache/{target_id}/{run_id}/workspace/" +``` + +- **Use for:** Fuzzing workflows that modify files (corpus, crashes) +- **Advantages:** Safe for concurrent execution, no file conflicts +- **Cleanup:** Entire run directory removed after workflow completes + +**2. Shared Mode** +```python +# All runs share the same workspace +cache_path = f"/cache/{target_id}/workspace/" +``` + +- **Use for:** Read-only analysis workflows (security scanning, static analysis) +- **Advantages:** Efficient (downloads once), lower bandwidth/storage +- **Cleanup:** No cleanup (workspace persists for reuse) + +**3. Copy-on-Write Mode** +```python +# Download once to shared location, copy per run +shared_cache = f"/cache/{target_id}/shared/workspace/" +run_cache = f"/cache/{target_id}/{run_id}/workspace/" +``` + +- **Use for:** Large targets that need isolation +- **Advantages:** Download once, isolated per-run execution +- **Cleanup:** Run-specific copies removed, shared cache persists + +**Configuration:** + +Workflows specify isolation mode in `metadata.yaml`: +```yaml +name: atheris_fuzzing +workspace_isolation: "isolated" # or "shared" or "copy-on-write" +``` + +Workers automatically handle download, extraction, and cleanup based on the mode. + +--- + +## Dynamic Workflow Loading + +### The Problem + +**Requirement:** Workflows must be dynamically added without modifying the codebase or rebuilding Docker images. + +**Traditional approach (doesn't work):** +- Build Docker image per workflow with dependencies +- Push to registry +- Worker pulls and spawns container +- āŒ Requires rebuild for every workflow change +- āŒ Registry overhead +- āŒ Slow (5-10s startup per workflow) + +**Our approach (works):** +- Workflow code mounted as volume into long-lived workers +- Workers scan `/app/toolbox/workflows` at startup +- Dynamically import and register workflows matching vertical +- āœ… No rebuild needed +- āœ… No registry +- āœ… Zero startup overhead + +### Implementation + +**1. Docker Compose volume mount:** +```yaml +worker-android: + volumes: + - ./toolbox:/app/toolbox:ro # Mount workflow code as read-only +``` + +**2. Worker discovers workflows:** +```python +# Runs at worker startup +for workflow_dir in Path("/app/toolbox/workflows").iterdir(): + metadata = yaml.safe_load((workflow_dir / "metadata.yaml").read_text()) + + # Only load workflows for this vertical + if metadata.get("vertical") == os.getenv("WORKER_VERTICAL"): + # Dynamically import workflow.py + module = importlib.import_module(f"toolbox.workflows.{workflow_dir.name}.workflow") + + # Find @workflow.defn classes + workflows.append(module.MyWorkflowClass) +``` + +**3. Developer adds workflow:** +```bash +# 1. Create workflow directory +mkdir -p toolbox/workflows/my_new_workflow + +# 2. Write metadata +cat > toolbox/workflows/my_new_workflow/metadata.yaml < toolbox/workflows/my_new_workflow/workflow.py <80%, memory >90%) + +### Phase 2: Multi-Host (6-18 months) + +**Configuration:** +``` +Host 1: Temporal + MinIO +Host 2: 5Ɨ worker-android +Host 3: 5Ɨ worker-rust +Host 4: 5Ɨ worker-web +``` + +**Changes required:** +```yaml +# Point all workers to central Temporal/MinIO +environment: + TEMPORAL_ADDRESS: temporal.prod.fuzzforge.ai:7233 + S3_ENDPOINT: http://minio.prod.fuzzforge.ai:9000 +``` + +**Capacity:** 3Ɨ Phase 1 = 45-150 concurrent workflows + +### Phase 3: Nomad Cluster (18+ months, if needed) + +**Trigger Points:** +- Managing 10+ hosts manually +- Need auto-scaling based on queue depth +- Need multi-tenancy (customer namespaces) + +**Migration effort:** 1-2 weeks (workers unchanged, just change deployment method) + +--- + +## File Lifecycle Management + +### Automatic Cleanup via MinIO Lifecycle Policies + +```bash +# Set on bucket (done by minio-setup service) +mc ilm add fuzzforge/targets --expiry-days 7 + +# MinIO automatically deletes objects older than 7 days +``` + +### Local Cache Eviction (LRU) + +```python +# Worker background task (runs every 30 minutes) +async def cleanup_cache_task(): + while True: + await storage.cleanup_cache() # LRU eviction + await asyncio.sleep(1800) # 30 minutes +``` + +### Manual Deletion (API) + +```python +@app.delete("/api/targets/{target_id}") +async def delete_target(target_id: str): + """Allow users to manually delete uploaded targets""" + s3.delete_object(Bucket='targets', Key=f'{target_id}/target') + return {"status": "deleted"} +``` + +### Retention Policies + +| Object Type | Default TTL | Configurable | Notes | +|-------------|-------------|--------------|-------| +| Uploaded targets | 7 days | Yes (env var) | Auto-deleted by MinIO | +| Worker cache | LRU (10GB limit) | Yes | Evicted when cache full | +| Workflow results | 30 days (optional) | Yes | Can store in MinIO | + +--- + +## Future: Nomad Migration + +### When to Add Nomad? + +**Trigger points:** +- Managing 10+ hosts manually becomes painful +- Need auto-scaling based on queue depth +- Need multi-tenancy with resource quotas +- Want sophisticated scheduling (bin-packing, affinity rules) + +**Estimated timing:** 18-24 months + +### Migration Complexity + +**Effort:** 1-2 weeks + +**What changes:** +- Deployment method (docker-compose ā†’ Nomad jobs) +- Orchestration layer (manual ā†’ Nomad scheduler) + +**What stays the same:** +- Worker Docker images (unchanged) +- Workflows (unchanged) +- Temporal (unchanged) +- MinIO (unchanged) +- Storage backend (unchanged) + +### Nomad Job Example + +```hcl +job "fuzzforge-worker-android" { + datacenters = ["dc1"] + type = "service" + + group "workers" { + count = 5 # Auto-scales based on queue depth + + scaling { + min = 1 + max = 20 + + policy { + evaluation_interval = "30s" + + check "queue_depth" { + source = "prometheus" + query = "temporal_queue_depth{queue='android-queue'}" + + strategy "target-value" { + target = 10 # Scale up if >10 tasks queued + } + } + } + } + + task "worker" { + driver = "docker" + + config { + image = "fuzzforge/worker-android:latest" + + volumes = [ + "/opt/fuzzforge/toolbox:/app/toolbox:ro" + ] + } + + env { + TEMPORAL_ADDRESS = "temporal.service.consul:7233" + WORKER_VERTICAL = "android" + S3_ENDPOINT = "http://minio.service.consul:9000" + } + + resources { + cpu = 500 # MHz + memory = 512 # MB + } + } + } +} +``` + +### Licensing Considerations + +**Nomad BSL 1.1 Risk:** Depends on FuzzForge positioning + +**Safe positioning (LOW risk):** +- āœ… Market as "Android/Rust/Web security verticals" +- āœ… Emphasize domain expertise, not orchestration +- āœ… Nomad is internal infrastructure +- āœ… Customers buy security services, not Nomad + +**Risky positioning (MEDIUM risk):** +- āš ļø Market as "generic workflow orchestration platform" +- āš ļø Emphasize flexibility over domain expertise +- āš ļø Could be seen as competing with HashiCorp + +**Mitigation:** +- Keep marketing focused on security verticals +- Get legal review before Phase 3 +- Alternative: Use Kubernetes (Apache 2.0, zero risk) + +--- + +## Migration Timeline + +### Phase 1: Foundation (Weeks 1-2) +- āœ… Create feature branch +- Set up Temporal docker-compose +- Add MinIO service +- Implement S3CachedStorage backend +- Create cleanup/lifecycle logic + +### Phase 2: First Vertical Worker (Weeks 3-4) +- Design worker base template +- Create worker-rust with AFL++, cargo-fuzz +- Implement dynamic workflow discovery +- Test workflow loading from mounted volume + +### Phase 3: Migrate Workflows (Weeks 5-6) +- Port security_assessment workflow to Temporal +- Update workflow metadata format +- Test end-to-end flow (upload ā†’ analyze ā†’ results) +- Verify cleanup/lifecycle + +### Phase 4: Additional Verticals (Weeks 7-8) +- Create worker-android, worker-web +- Document vertical development guide +- Update CLI for MinIO uploads +- Update backend API for Temporal + +### Phase 5: Testing & Docs (Weeks 9-10) +- Comprehensive testing +- Update README +- Migration guide for existing users +- Troubleshooting documentation + +**Total: 10 weeks, rollback possible at any phase** + +--- + +## Decision Log + +### 2025-09-30: Architecture Implementation +- **Decision:** Temporal with Vertical Workers +- **Rationale:** Simpler infrastructure, better reliability, clear scaling path + +### 2025-10-01: Vertical Worker Model +- **Decision:** Use long-lived vertical workers instead of ephemeral per-workflow containers +- **Rationale:** + - Zero startup overhead (5s saved per workflow) + - Pre-built toolchains (Android, Rust, Web, etc.) + - Dynamic workflows via mounted volumes (no image rebuild) + - Better marketing (sell verticals, not orchestration) + - Safer Nomad BSL positioning + +### 2025-10-01: Unified MinIO Storage +- **Decision:** Use MinIO for both dev and production (no LocalVolumeStorage) +- **Rationale:** + - Unified codebase (no environment-specific code) + - Lightweight (256MB with CI_CD=true) + - Negligible overhead (2-4s for 250MB upload) + - Better security (no host filesystem mounts) + - Multi-host ready + - Automatic cleanup via lifecycle policies + +### 2025-10-01: Dynamic Workflow Loading +- **Decision:** Mount workflow code as volume, discover at runtime +- **Rationale:** + - Add workflows without rebuilding images + - No registry overhead + - Supports user-contributed workflows + - Faster iteration for developers + +--- + +**Document Version:** 2.0 +**Last Updated:** 2025-10-01 +**Next Review:** After Phase 1 implementation (2 weeks) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..649d8fb --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,85 @@ +# Changelog + +All notable changes to FuzzForge will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.7.0] - 2025-01-16 + +### šŸŽÆ Major Features + +#### Secret Detection Workflows +- **Added three secret detection workflows**: + - `gitleaks_detection` - Pattern-based secret scanning + - `trufflehog_detection` - Entropy-based secret detection with verification + - `llm_secret_detection` - AI-powered semantic secret detection using LLMs +- **Comprehensive benchmarking infrastructure**: + - 32-secret ground truth dataset for precision/recall testing + - Difficulty levels: 12 Easy, 10 Medium, 10 Hard secrets + - SARIF-formatted output for all workflows + - Achieved 100% recall with LLM-based detection on benchmark dataset + +#### AI Module & Agent Integration +- Added A2A (Agent-to-Agent) wrapper for multi-agent orchestration +- Task agent implementation with Google ADK +- LLM analysis workflow for code security analysis +- Reactivated AI agent command (`ff ai agent`) + +#### Temporal Migration Complete +- Fully migrated from Prefect to Temporal for workflow orchestration +- MinIO storage for unified file handling (replaces volume mounts) +- Vertical workers with pre-built security toolchains +- Improved worker lifecycle management + +#### CI/CD Integration +- Ephemeral deployment model for testing +- Automated workflow validation in CI pipeline + +### āœØ Enhancements + +#### Documentation +- Updated README for Temporal + MinIO architecture +- Removed obsolete `volume_mode` references across all documentation +- Added `.env` configuration guide for AI agent API keys +- Fixed worker startup instructions with correct service names +- Updated docker compose commands to modern syntax + +#### Worker Management +- Added `worker_service` field to API responses for correct service naming +- Improved error messages with actionable manual start commands +- Fixed default parameters for gitleaks (now uses `no_git=True` by default) + +### šŸ› Bug Fixes + +- Fixed gitleaks workflow failing on uploaded directories without Git history +- Fixed worker startup command suggestions (now uses `docker compose up -d` with service names) +- Fixed missing `cognify_text` method in CogneeProjectIntegration + +### šŸ”§ Technical Changes + +- Updated all package versions to 0.7.0 +- Improved SARIF output formatting for secret detection workflows +- Enhanced benchmark validation with ground truth JSON +- Better integration between CLI and backend for worker management + +### šŸ“ Test Projects + +- Added `secret_detection_benchmark` with 32 documented secrets +- Ground truth JSON for automated precision/recall calculations +- Updated `vulnerable_app` for comprehensive security testing + +--- + +## [0.6.0] - 2024-12-XX + +### Features +- Initial Temporal migration +- Fuzzing workflows (Atheris, Cargo, OSS-Fuzz) +- Security assessment workflow +- Basic CLI commands + +--- + +[0.7.0]: https://github.com/FuzzingLabs/fuzzforge_ai/compare/v0.6.0...v0.7.0 +[0.6.0]: https://github.com/FuzzingLabs/fuzzforge_ai/releases/tag/v0.6.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e948db..6529eec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,9 +84,10 @@ docs(readme): update installation instructions ``` backend/toolbox/workflows/your_workflow/ ā”œā”€ā”€ __init__.py - ā”œā”€ā”€ workflow.py # Main Prefect flow - ā”œā”€ā”€ metadata.yaml # Workflow metadata - ā””ā”€ā”€ Dockerfile # Container definition + ā”œā”€ā”€ workflow.py # Main Temporal workflow + ā”œā”€ā”€ activities.py # Workflow activities (optional) + ā”œā”€ā”€ metadata.yaml # Workflow metadata (includes vertical field) + ā””ā”€ā”€ requirements.txt # Additional dependencies (optional) ``` 2. **Register Your Workflow** diff --git a/QUICKSTART_TEMPORAL.md b/QUICKSTART_TEMPORAL.md new file mode 100644 index 0000000..4264037 --- /dev/null +++ b/QUICKSTART_TEMPORAL.md @@ -0,0 +1,421 @@ +# FuzzForge Temporal Architecture - Quick Start Guide + +This guide walks you through starting and testing the new Temporal-based architecture. + +## Prerequisites + +- Docker and Docker Compose installed +- At least 2GB free RAM (core services only, workers start on-demand) +- Ports available: 7233, 8233, 9000, 9001, 8000 + +## Step 1: Start Core Services + +```bash +# From project root +cd /path/to/fuzzforge_ai + +# Start core services (Temporal, MinIO, Backend) +docker-compose up -d + +# Workers are pre-built but don't auto-start (saves ~6-7GB RAM) +# They'll start automatically when workflows need them + +# Check status +docker-compose ps +``` + +**Expected output:** +``` +NAME STATUS PORTS +fuzzforge-minio healthy 0.0.0.0:9000-9001->9000-9001/tcp +fuzzforge-temporal healthy 0.0.0.0:7233->7233/tcp +fuzzforge-temporal-postgresql healthy 5432/tcp +fuzzforge-backend healthy 0.0.0.0:8000->8000/tcp +fuzzforge-minio-setup exited (0) +# Workers NOT running (will start on-demand) +``` + +**First startup takes ~30-60 seconds** for health checks to pass. + +## Step 2: Verify Worker Discovery + +Check worker logs to ensure workflows are discovered: + +```bash +docker logs fuzzforge-worker-rust +``` + +**Expected output:** +``` +============================================================ +FuzzForge Vertical Worker: rust +============================================================ +Temporal Address: temporal:7233 +Task Queue: rust-queue +Max Concurrent Activities: 5 +============================================================ +Discovering workflows for vertical: rust +Importing workflow module: toolbox.workflows.rust_test.workflow +āœ“ Discovered workflow: RustTestWorkflow from rust_test (vertical: rust) +Discovered 1 workflows for vertical 'rust' +Connecting to Temporal at temporal:7233... +āœ“ Connected to Temporal successfully +Creating worker on task queue: rust-queue +āœ“ Worker created successfully +============================================================ +šŸš€ Worker started for vertical 'rust' +šŸ“¦ Registered 1 workflows +āš™ļø Registered 3 activities +šŸ“Ø Listening on task queue: rust-queue +============================================================ +Worker is ready to process tasks... +``` + +## Step 2.5: Worker Lifecycle Management (New in v0.7.0) + +Workers start on-demand when workflows need them: + +```bash +# Check worker status (should show Exited or not running) +docker ps -a --filter "name=fuzzforge-worker" + +# Run a workflow - worker starts automatically +ff workflow run ossfuzz_campaign . project_name=zlib + +# Worker is now running +docker ps --filter "name=fuzzforge-worker-ossfuzz" +``` + +**Configuration** (`.fuzzforge/config.yaml`): +```yaml +workers: + auto_start_workers: true # Default: auto-start + auto_stop_workers: false # Default: keep running + worker_startup_timeout: 60 # Startup timeout in seconds +``` + +**CLI Control**: +```bash +# Disable auto-start +ff workflow run ossfuzz_campaign . --no-auto-start + +# Enable auto-stop after completion +ff workflow run ossfuzz_campaign . --wait --auto-stop +``` + +## Step 3: Access Web UIs + +### Temporal Web UI +- URL: http://localhost:8233 +- View workflows, executions, and task queues + +### MinIO Console +- URL: http://localhost:9001 +- Login: `fuzzforge` / `fuzzforge123` +- View uploaded targets and results + +## Step 4: Test Workflow Execution + +### Option A: Using Temporal CLI (tctl) + +```bash +# Install tctl (if not already installed) +brew install temporal # macOS +# or download from https://github.com/temporalio/tctl/releases + +# Execute test workflow +tctl workflow run \ + --address localhost:7233 \ + --taskqueue rust-queue \ + --workflow_type RustTestWorkflow \ + --input '{"target_id": "test-123", "test_message": "Hello Temporal!"}' +``` + +### Option B: Using Python Client + +Create `test_workflow.py`: + +```python +import asyncio +from temporalio.client import Client + +async def main(): + # Connect to Temporal + client = await Client.connect("localhost:7233") + + # Start workflow + result = await client.execute_workflow( + "RustTestWorkflow", + {"target_id": "test-123", "test_message": "Hello Temporal!"}, + id="test-workflow-1", + task_queue="rust-queue" + ) + + print("Workflow result:", result) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +```bash +python test_workflow.py +``` + +### Option C: Upload Target and Run (Full Flow) + +```python +# upload_and_run.py +import asyncio +import boto3 +from pathlib import Path +from temporalio.client import Client + +async def main(): + # 1. Upload target to MinIO + s3 = boto3.client( + 's3', + endpoint_url='http://localhost:9000', + aws_access_key_id='fuzzforge', + aws_secret_access_key='fuzzforge123', + region_name='us-east-1' + ) + + # Create a test file + test_file = Path('/tmp/test_target.txt') + test_file.write_text('This is a test target file') + + # Upload to MinIO + target_id = 'my-test-target-001' + s3.upload_file( + str(test_file), + 'targets', + f'{target_id}/target' + ) + print(f"āœ“ Uploaded target: {target_id}") + + # 2. Run workflow + client = await Client.connect("localhost:7233") + + result = await client.execute_workflow( + "RustTestWorkflow", + {"target_id": target_id, "test_message": "Full flow test!"}, + id=f"workflow-{target_id}", + task_queue="rust-queue" + ) + + print("āœ“ Workflow completed!") + print("Results:", result) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +```bash +# Install dependencies +pip install temporalio boto3 + +# Run test +python upload_and_run.py +``` + +## Step 5: Monitor Execution + +### View in Temporal UI + +1. Open http://localhost:8233 +2. Click on "Workflows" +3. Find your workflow by ID +4. Click to see: + - Execution history + - Activity results + - Error stack traces (if any) + +### View Logs + +```bash +# Worker logs (shows activity execution) +docker logs -f fuzzforge-worker-rust + +# Temporal server logs +docker logs -f fuzzforge-temporal +``` + +### Check MinIO Storage + +1. Open http://localhost:9001 +2. Login: `fuzzforge` / `fuzzforge123` +3. Browse buckets: + - `targets/` - Uploaded target files + - `results/` - Workflow results (if uploaded) + - `cache/` - Worker cache (temporary) + +## Troubleshooting + +### Services Not Starting + +```bash +# Check logs for all services +docker-compose -f docker-compose.temporal.yaml logs + +# Check specific service +docker-compose -f docker-compose.temporal.yaml logs temporal +docker-compose -f docker-compose.temporal.yaml logs minio +docker-compose -f docker-compose.temporal.yaml logs worker-rust +``` + +### Worker Not Discovering Workflows + +**Issue**: Worker logs show "No workflows found for vertical: rust" + +**Solution**: +1. Check toolbox mount: `docker exec fuzzforge-worker-rust ls /app/toolbox/workflows` +2. Verify metadata.yaml exists and has `vertical: rust` +3. Check workflow.py has `@workflow.defn` decorator + +### Cannot Connect to Temporal + +**Issue**: `Failed to connect to Temporal` + +**Solution**: +```bash +# Wait for Temporal to be healthy +docker-compose -f docker-compose.temporal.yaml ps + +# Check Temporal health manually +curl http://localhost:8233 + +# Restart Temporal if needed +docker-compose -f docker-compose.temporal.yaml restart temporal +``` + +### MinIO Connection Failed + +**Issue**: `Failed to download target` + +**Solution**: +```bash +# Check MinIO is running +docker ps | grep minio + +# Check buckets exist +docker exec fuzzforge-minio mc ls fuzzforge/ + +# Verify target was uploaded +docker exec fuzzforge-minio mc ls fuzzforge/targets/ +``` + +### Workflow Hangs + +**Issue**: Workflow starts but never completes + +**Check**: +1. Worker logs for errors: `docker logs fuzzforge-worker-rust` +2. Activity timeouts in workflow code +3. Target file actually exists in MinIO + +## Scaling + +### Add More Workers + +```bash +# Scale rust workers horizontally +docker-compose -f docker-compose.temporal.yaml up -d --scale worker-rust=3 + +# Verify all workers are running +docker ps | grep worker-rust +``` + +### Increase Concurrent Activities + +Edit `docker-compose.temporal.yaml`: + +```yaml +worker-rust: + environment: + MAX_CONCURRENT_ACTIVITIES: 10 # Increase from 5 +``` + +```bash +# Apply changes +docker-compose -f docker-compose.temporal.yaml up -d worker-rust +``` + +## Cleanup + +```bash +# Stop all services +docker-compose -f docker-compose.temporal.yaml down + +# Remove volumes (WARNING: deletes all data) +docker-compose -f docker-compose.temporal.yaml down -v + +# Remove everything including images +docker-compose -f docker-compose.temporal.yaml down -v --rmi all +``` + +## Next Steps + +1. **Add More Workflows**: Create workflows in `backend/toolbox/workflows/` +2. **Add More Verticals**: Create new worker types (android, web, etc.) - see `workers/README.md` +3. **Integrate with Backend**: Update FastAPI backend to use Temporal client +4. **Update CLI**: Modify `ff` CLI to work with Temporal workflows + +## Useful Commands + +```bash +# View all logs +docker-compose -f docker-compose.temporal.yaml logs -f + +# View specific service logs +docker-compose -f docker-compose.temporal.yaml logs -f worker-rust + +# Restart a service +docker-compose -f docker-compose.temporal.yaml restart worker-rust + +# Check service status +docker-compose -f docker-compose.temporal.yaml ps + +# Execute command in worker +docker exec -it fuzzforge-worker-rust bash + +# View worker Python environment +docker exec fuzzforge-worker-rust pip list + +# Check workflow discovery manually +docker exec fuzzforge-worker-rust python -c " +from pathlib import Path +import yaml +for w in Path('/app/toolbox/workflows').iterdir(): + if w.is_dir(): + meta = w / 'metadata.yaml' + if meta.exists(): + print(f'{w.name}: {yaml.safe_load(meta.read_text()).get(\"vertical\")}')" +``` + +## Architecture Overview + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Temporal ā”‚ā”€ā”€ā”€ā”€ā–¶ā”‚ Task Queue ā”‚ā”€ā”€ā”€ā”€ā–¶ā”‚ Worker-Rust ā”‚ +ā”‚ Server ā”‚ ā”‚ rust-queue ā”‚ ā”‚ (Long-lived)ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ ā”‚ + ā”‚ ā”‚ + ā–¼ ā–¼ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Postgres ā”‚ ā”‚ MinIO ā”‚ +ā”‚ (State) ā”‚ ā”‚ (Storage) ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā”‚ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā” + ā”‚ ā”‚ + ā”Œā”€ā”€ā”€ā”€ā–¼ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā–¼ā”€ā”€ā”€ā”€ā” + ā”‚ Targets ā”‚ ā”‚ Results ā”‚ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ +``` + +## Support + +- **Documentation**: See `ARCHITECTURE.md` for detailed design +- **Worker Guide**: See `workers/README.md` for adding verticals +- **Issues**: Open GitHub issue with logs and steps to reproduce diff --git a/README.md b/README.md index 89c5cb2..b88eeff 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ License: BSL + Apache Python 3.11+ Website - Version + Version GitHub Stars

@@ -41,6 +41,8 @@ FuzzForge is **open source**, built to empower security teams, researchers, and the community. > šŸš§ FuzzForge is under active development. Expect breaking changes. +> +> **Note:** Fuzzing workflows (`atheris_fuzzing`, `cargo_fuzzing`, `ossfuzz_campaign`) are in early development. OSS-Fuzz integration is under heavy active development. For stable workflows, use: `security_assessment`, `gitleaks_detection`, `trufflehog_detection`, or `llm_secret_detection`. --- @@ -59,12 +61,29 @@ If you find FuzzForge useful, please star the repo to support development šŸš€ - šŸ¤– **AI Agents for Security** ā€“ Specialized agents for AppSec, reversing, and fuzzing - šŸ›  **Workflow Automation** ā€“ Define & execute AppSec workflows as code - šŸ“ˆ **Vulnerability Research at Scale** ā€“ Rediscover 1-days & find 0-days with automation -- šŸ”— **Fuzzer Integration** ā€“ AFL, Honggfuzz, AFLnet, StateAFL & more +- šŸ”— **Fuzzer Integration** ā€“ Atheris (Python), cargo-fuzz (Rust), OSS-Fuzz campaigns - šŸŒ **Community Marketplace** ā€“ Share workflows, corpora, PoCs, and modules - šŸ”’ **Enterprise Ready** ā€“ Team/Corp cloud tiers for scaling offensive security --- +## šŸ” Secret Detection Benchmarks + +FuzzForge includes three secret detection workflows benchmarked on a controlled dataset of **32 documented secrets** (12 Easy, 10 Medium, 10 Hard): + +| Tool | Recall | Secrets Found | Speed | +|------|--------|---------------|-------| +| **LLM (gpt-5-mini)** | **84.4%** | 41 | 618s | +| **LLM (gpt-4o-mini)** | 56.2% | 30 | 297s | +| **Gitleaks** | 37.5% | 12 | 5s | +| **TruffleHog** | 0.0% | 1 | 5s | + +šŸ“Š [Full benchmark results and analysis](backend/benchmarks/by_category/secret_detection/results/comparison_report.md) + +The LLM-based detector excels at finding obfuscated and hidden secrets through semantic analysis, while pattern-based tools (Gitleaks) offer speed for standard secret formats. + +--- + ## šŸ“¦ Installation ### Requirements @@ -81,38 +100,20 @@ curl -LsSf https://astral.sh/uv/install.sh | sh **Docker** For containerized workflows, see the [Docker Installation Guide](https://docs.docker.com/get-docker/). -#### Configure Docker Daemon +#### Configure AI Agent API Keys (Optional) -Before running `docker compose up`, configure Docker to allow insecure registries (required for the local registry). +For AI-powered workflows, configure your LLM API keys: -Add the following to your Docker daemon configuration: - -```json -{ - "insecure-registries": [ - "localhost:5000", - "host.docker.internal:5001", - "registry:5000" - ] -} +```bash +cp volumes/env/.env.example volumes/env/.env +# Edit volumes/env/.env and add your API keys (OpenAI, Anthropic, Google, etc.) ``` -**macOS (Docker Desktop):** -1. Open Docker Desktop -2. Go to Settings ā†’ Docker Engine -3. Add the `insecure-registries` configuration to the JSON -4. Click "Apply & Restart" +This is required for: +- `llm_secret_detection` workflow +- AI agent features (`ff ai agent`) -**Linux:** -1. Edit `/etc/docker/daemon.json` (create if it doesn't exist): - ```bash - sudo nano /etc/docker/daemon.json - ``` -2. Add the configuration above -3. Restart Docker: - ```bash - sudo systemctl restart docker - ``` +Basic security workflows (gitleaks, trufflehog, security_assessment) work without this configuration. ### CLI Installation @@ -131,31 +132,38 @@ uv tool install --python python3.12 . ## āš” Quickstart -Run your first workflow : +Run your first workflow with **Temporal orchestration** and **automatic file upload**: ```bash # 1. Clone the repo git clone https://github.com/fuzzinglabs/fuzzforge_ai.git cd fuzzforge_ai -# 2. Build & run with Docker -# Set registry host for your OS (local registry is mandatory) -# macOS/Windows (Docker Desktop): -export REGISTRY_HOST=host.docker.internal -# Linux (default): -# export REGISTRY_HOST=localhost +# 2. Start FuzzForge with Temporal docker compose up -d ``` -> The first launch can take 5-10 minutes due to Docker image building - a good time for a coffee break ā˜• +> The first launch can take 2-3 minutes for services to initialize ā˜• ```bash -# 3. Run your first workflow -cd test_projects/vulnerable_app/ # Go into the test directory -fuzzforge init # Init a fuzzforge project -ff workflow run security_assessment . # Start a workflow (you can also use ff command) +# 3. Run your first workflow (files are automatically uploaded) +cd test_projects/vulnerable_app/ +fuzzforge init # Initialize FuzzForge project +ff workflow run security_assessment . # Start workflow - CLI uploads files automatically! + +# The CLI will: +# - Detect the local directory +# - Create a compressed tarball +# - Upload to backend (via MinIO) +# - Start the workflow on vertical worker ``` +**What's running:** +- **Temporal**: Workflow orchestration (UI at http://localhost:8233) +- **MinIO**: File storage for targets (Console at http://localhost:9001) +- **Vertical Workers**: Pre-built workers with security toolchains +- **Backend API**: FuzzForge REST API (http://localhost:8000) + ### Manual Workflow Setup ![Manual Workflow Demo](docs/static/videos/manual_workflow.gif) diff --git a/ai/README.md b/ai/README.md index 36f1f2f..254fdd2 100644 --- a/ai/README.md +++ b/ai/README.md @@ -1,6 +1,6 @@ # FuzzForge AI Module -FuzzForge AI is the multi-agent layer that lets you operate the FuzzForge security platform through natural language. It orchestrates local tooling, registered Agent-to-Agent (A2A) peers, and the Prefect-powered backend while keeping long-running context in memory and project knowledge graphs. +FuzzForge AI is the multi-agent layer that lets you operate the FuzzForge security platform through natural language. It orchestrates local tooling, registered Agent-to-Agent (A2A) peers, and the Temporal-powered backend while keeping long-running context in memory and project knowledge graphs. ## Quick Start @@ -32,7 +32,7 @@ FuzzForge AI is the multi-agent layer that lets you operate the FuzzForge securi ```bash fuzzforge ai agent ``` - Keep the backend running (Prefect API at `FUZZFORGE_MCP_URL`) so workflow commands succeed. + Keep the backend running (Temporal API at `FUZZFORGE_MCP_URL`) so workflow commands succeed. ## Everyday Workflow @@ -61,7 +61,7 @@ Inside `fuzzforge ai agent` you can mix slash commands and free-form prompts: /sendfile SecurityAgent src/report.md "Please review" You> route_to SecurityAnalyzer: scan ./backend for secrets You> run fuzzforge workflow static_analysis_scan on ./test_projects/demo -You> search project knowledge for "prefect status" using INSIGHTS +You> search project knowledge for "temporal status" using INSIGHTS ``` Artifacts created during the conversation are served from `.fuzzforge/artifacts/` and exposed through the A2A HTTP API. @@ -84,7 +84,7 @@ Use these to validate the setup once the agent shell is running: - `run fuzzforge workflow static_analysis_scan on ./backend with target_branch=main` - `show findings for that run once it finishes` - `refresh the project knowledge graph for ./backend` -- `search project knowledge for "prefect readiness" using INSIGHTS` +- `search project knowledge for "temporal readiness" using INSIGHTS` - `/recall terraform secrets` - `/memory status` - `ROUTE_TO SecurityAnalyzer: audit infrastructure_vulnerable` diff --git a/ai/agents/task_agent/.dockerignore b/ai/agents/task_agent/.dockerignore new file mode 100644 index 0000000..227dc09 --- /dev/null +++ b/ai/agents/task_agent/.dockerignore @@ -0,0 +1,9 @@ +__pycache__ +*.pyc +*.pyo +*.pytest_cache +*.coverage +coverage.xml +build/ +dist/ +.env diff --git a/ai/agents/task_agent/.env.example b/ai/agents/task_agent/.env.example new file mode 100644 index 0000000..c71d59a --- /dev/null +++ b/ai/agents/task_agent/.env.example @@ -0,0 +1,10 @@ +# Default LiteLLM configuration +LITELLM_MODEL=gemini/gemini-2.0-flash-001 +# LITELLM_PROVIDER=gemini + +# API keys (uncomment and fill as needed) +# GOOGLE_API_KEY= +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= +# OPENROUTER_API_KEY= +# MISTRAL_API_KEY= diff --git a/ai/agents/task_agent/ARCHITECTURE.md b/ai/agents/task_agent/ARCHITECTURE.md new file mode 100644 index 0000000..2210dca --- /dev/null +++ b/ai/agents/task_agent/ARCHITECTURE.md @@ -0,0 +1,82 @@ +# Architecture Overview + +This package is a minimal ADK agent that keeps runtime behaviour and A2A access in separate layers so it can double as boilerplate. + +## Directory Layout + +```text +agent_with_adk_format/ +ā”œā”€ā”€ __init__.py # Exposes root_agent for ADK runners +ā”œā”€ā”€ a2a_hot_swap.py # JSON-RPC helper for model/prompt swaps +ā”œā”€ā”€ README.md, QUICKSTART.md # Operational docs +ā”œā”€ā”€ ARCHITECTURE.md # This document +ā”œā”€ā”€ .env # Active environment (gitignored) +ā”œā”€ā”€ .env.example # Environment template +ā””ā”€ā”€ litellm_agent/ + ā”œā”€ā”€ agent.py # Root Agent definition (LiteLLM shell) + ā”œā”€ā”€ callbacks.py # before_agent / before_model hooks + ā”œā”€ā”€ config.py # Defaults, state keys, control prefix + ā”œā”€ā”€ control.py # HOTSWAP command parsing/serialization + ā”œā”€ā”€ state.py # Session state wrapper + LiteLLM factory + ā”œā”€ā”€ tools.py # set_model / set_prompt / get_config + ā”œā”€ā”€ prompts.py # Base instruction text + ā””ā”€ā”€ agent.json # A2A agent card (served under /.well-known) +``` + +```mermaid +flowchart TD + subgraph ADK Runner + A["adk api_server / adk web / adk run"] + B["agent_with_adk_format/__init__.py"] + C["litellm_agent/agent.py (root_agent)"] + D["HotSwapState (state.py)"] + E["LiteLlm(model, provider)"] + end + + subgraph Session State + S1[app:litellm_agent/model] + S2[app:litellm_agent/provider] + S3[app:litellm_agent/prompt] + end + + A --> B --> C + C --> D + D -->|instantiate| E + D --> S1 + D --> S2 + D --> S3 + E --> C +``` + +## Runtime Flow (ADK Runners) + +1. **Startup**: `adk api_server`/`adk web` imports `agent_with_adk_format`, which exposes `root_agent` from `litellm_agent/agent.py`. `.env` at package root is loaded before the runner constructs the agent. +2. **Session State**: `callbacks.py` and `tools.py` read/write through `state.py`. We store `model`, `provider`, and `prompt` keys (prefixed `app:litellm_agent/ā€¦`) which persist across turns. +3. **Instruction Generation**: `provide_instruction` composes the base persona from `prompts.py` plus any stored prompt override. The current model/provider is appended for observability. +4. **Model Hot-Swap**: When a control message is detected (`[HOTSWAP:MODEL:ā€¦]`) the callback parses it via `control.py`, updates the session state, and calls `state.apply_state_to_agent` to instantiate a new `LiteLlm(model=ā€¦, custom_llm_provider=ā€¦)`. ADK runners reuse that instance for subsequent turns. +5. **Prompt Hot-Swap**: Similar path (`set_prompt` tool/callback) updates state; the dynamic instruction immediately reflects the change. +6. **Config Reporting**: Both the callback and the tool surface the summary string produced by `HotSwapState.describe()`, ensuring CLI, A2A, and UI all show the same data. + +## A2A Integration + +- `agent.json` defines the agent card and enables ADK to register `/a2a/litellm_agent` routes when launched with `--a2a`. +- `a2a_hot_swap.py` uses `a2a.client.A2AClient` to programmatically send control messages and user text via JSON-RPC. It supports streaming when available and falls back to blocking requests otherwise. + +```mermaid +sequenceDiagram + participant Client as a2a_hot_swap.py + participant Server as ADK API Server + participant Agent as root_agent + + Client->>Server: POST /a2a/litellm_agent (message/stream or message/send) + Server->>Agent: Invoke callbacks/tools + Agent->>Server: Status / artifacts / final message + Server->>Client: Streamed Task events + Client->>Client: Extract text & print summary +``` + +## Extending the Boilerplate + +- Add tools under `litellm_agent/tools.py` and register them in `agent.py` to expose new capabilities. +- Use `state.py` to track additional configuration or session data (store under your own prefix to avoid collisions). +- When layering business logic, prefer expanding callbacks or adding higher-level agents while leaving the hot-swap mechanism untouched for reuse. diff --git a/ai/agents/task_agent/DEPLOY.md b/ai/agents/task_agent/DEPLOY.md new file mode 100644 index 0000000..bf4d24c --- /dev/null +++ b/ai/agents/task_agent/DEPLOY.md @@ -0,0 +1,71 @@ +# Docker & Kubernetes Deployment + +## Local Docker + +Build from the repository root: + +```bash +docker build -t litellm-hot-swap:latest agent_with_adk_format +``` + +Run the container (port 8000, inject provider keys via env file or flags): + +```bash +docker run \ + -p 8000:8000 \ + --env-file agent_with_adk_format/.env \ + litellm-hot-swap:latest +``` + +The container serves Uvicorn on `http://localhost:8000`. Update `.env` (or pass `-e KEY=value`) before launching if you plan to hot-swap providers. + +## Kubernetes (example manifest) + +Use the same image, optionally pushed to a registry (`docker tag` + `docker push`). A simple Deployment/Service pair: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-hot-swap +spec: + replicas: 1 + selector: + matchLabels: + app: litellm-hot-swap + template: + metadata: + labels: + app: litellm-hot-swap + spec: + containers: + - name: server + image: /litellm-hot-swap:latest + ports: + - containerPort: 8000 + env: + - name: PORT + value: "8000" + - name: LITELLM_MODEL + value: gemini/gemini-2.0-flash-001 + # Add provider keys as needed + # - name: OPENAI_API_KEY + # valueFrom: + # secretKeyRef: + # name: litellm-secrets + # key: OPENAI_API_KEY +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm-hot-swap +spec: + type: LoadBalancer + selector: + app: litellm-hot-swap + ports: + - port: 80 + targetPort: 8000 +``` + +Apply with `kubectl apply -f deployment.yaml`. Provide secrets via `env` or Kubernetes Secrets. diff --git a/ai/agents/task_agent/Dockerfile b/ai/agents/task_agent/Dockerfile new file mode 100644 index 0000000..eaf734b --- /dev/null +++ b/ai/agents/task_agent/Dockerfile @@ -0,0 +1,19 @@ +# syntax=docker/dockerfile:1 + +FROM python:3.11-slim AS base + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PORT=8000 + +WORKDIR /app + +COPY requirements.txt ./requirements.txt +RUN pip install --upgrade pip && pip install -r requirements.txt + +COPY . /app/agent_with_adk_format +WORKDIR /app/agent_with_adk_format +ENV PYTHONPATH=/app + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/ai/agents/task_agent/QUICKSTART.md b/ai/agents/task_agent/QUICKSTART.md new file mode 100644 index 0000000..756a054 --- /dev/null +++ b/ai/agents/task_agent/QUICKSTART.md @@ -0,0 +1,61 @@ +# Quick Start Guide + +## Launch the Agent + +From the repository root you can expose the agent through any ADK entry point: + +```bash +# A2A / HTTP server +adk api_server --a2a --port 8000 agent_with_adk_format + +# Browser UI +adk web agent_with_adk_format + +# Interactive terminal +adk run agent_with_adk_format +``` + +The A2A server exposes the JSON-RPC endpoint at `http://localhost:8000/a2a/litellm_agent`. + +## Hot-Swap from the Command Line + +Use the bundled helper to change model and prompt via A2A without touching the UI: + +```bash +python agent_with_adk_format/a2a_hot_swap.py \ + --model openai gpt-4o \ + --prompt "You are concise." \ + --config \ + --context demo-session +``` + +The script sends the control messages for you and prints the serverā€™s responses. The `--context` flag lets you reuse the same conversation across multiple invocations. + +### Follow-up Messages + +Once the swaps are applied you can send a user message on the same session: + +```bash +python agent_with_adk_format/a2a_hot_swap.py \ + --context demo-session \ + --message "Summarise the current configuration in five words." +``` + +### Clearing the Prompt + +```bash +python agent_with_adk_format/a2a_hot_swap.py \ + --context demo-session \ + --prompt "" \ + --config +``` + +## Control Messages (for reference) + +Behind the scenes the helper sends plain text messages understood by the callbacks: + +- `[HOTSWAP:MODEL:provider/model]` +- `[HOTSWAP:PROMPT:text]` +- `[HOTSWAP:GET_CONFIG]` + +You can craft the same messages from any A2A client if you prefer. diff --git a/ai/agents/task_agent/README.md b/ai/agents/task_agent/README.md new file mode 100644 index 0000000..769ce33 --- /dev/null +++ b/ai/agents/task_agent/README.md @@ -0,0 +1,349 @@ +# LiteLLM Agent with Hot-Swap Support + +A flexible AI agent powered by LiteLLM that supports runtime hot-swapping of models and system prompts. Compatible with ADK and A2A protocols. + +## Features + +- šŸ”„ **Hot-Swap Models**: Change LLM models on-the-fly without restarting +- šŸ“ **Dynamic Prompts**: Update system prompts during conversation +- šŸŒ **Multi-Provider Support**: Works with OpenAI, Anthropic, Google, OpenRouter, and more +- šŸ”Œ **A2A Compatible**: Can be served as an A2A agent +- šŸ› ļø **ADK Integration**: Run with `adk web`, `adk run`, or `adk api_server` + +## Architecture + +``` +task_agent/ +ā”œā”€ā”€ __init__.py # Exposes root_agent for ADK +ā”œā”€ā”€ a2a_hot_swap.py # JSON-RPC helper for hot-swapping +ā”œā”€ā”€ README.md # This guide +ā”œā”€ā”€ QUICKSTART.md # Quick-start walkthrough +ā”œā”€ā”€ .env # Active environment (gitignored) +ā”œā”€ā”€ .env.example # Environment template +ā””ā”€ā”€ litellm_agent/ + ā”œā”€ā”€ __init__.py + ā”œā”€ā”€ agent.py # Main agent implementation + ā”œā”€ā”€ agent.json # A2A agent card + ā”œā”€ā”€ callbacks.py # ADK callbacks + ā”œā”€ā”€ config.py # Defaults and state keys + ā”œā”€ā”€ control.py # HOTSWAP message helpers + ā”œā”€ā”€ prompts.py # Base instruction + ā”œā”€ā”€ state.py # Session state utilities + ā””ā”€ā”€ tools.py # set_model / set_prompt / get_config +``` + +## Setup + +### 1. Environment Configuration + +Copying the example file is optionalā€”the repository already ships with a root-level `.env` seeded with defaults. Adjust the values at the package root: +```bash +cd task_agent +# Optionally refresh from the template +# cp .env.example .env +``` + +Edit `.env` (or `.env.example`) and add your API keys. The agent must be restarted after changes so the values are picked up: +```bash +# Set default model +LITELLM_MODEL=gemini/gemini-2.0-flash-001 + +# Add API keys for providers you want to use +GOOGLE_API_KEY=your_google_api_key +OPENAI_API_KEY=your_openai_api_key +ANTHROPIC_API_KEY=your_anthropic_api_key +OPENROUTER_API_KEY=your_openrouter_api_key +``` + +### 2. Install Dependencies + +```bash +pip install "google-adk" "a2a-sdk[all]" "python-dotenv" "litellm" +``` + +### 3. Run in Docker + +Build the container (this image can be pushed to any registry or run locally): + +```bash +docker build -t litellm-hot-swap:latest task_agent +``` + +Provide environment configuration at runtime (either pass variables individually or mount a file): + +```bash +docker run \ + -p 8000:8000 \ + --env-file task_agent/.env \ + litellm-hot-swap:latest +``` + +The container starts Uvicorn with the ADK app (`main.py`) listening on port 8000. + +## Running the Agent + +### Option 1: ADK Web UI (Recommended for Testing) + +Start the web interface: +```bash +adk web task_agent +``` + +> **Tip:** before launching `adk web`/`adk run`/`adk api_server`, ensure the root-level `.env` contains valid API keys for any provider you plan to hot-swap to (e.g. set `OPENAI_API_KEY` before switching to `openai/gpt-4o`). + +Open http://localhost:8000 in your browser and interact with the agent. + +### Option 2: ADK Terminal + +Run in terminal mode: +```bash +adk run task_agent +``` + +### Option 3: A2A API Server + +Start as an A2A-compatible API server: +```bash +adk api_server --a2a --port 8000 task_agent +``` + +The agent will be available at: `http://localhost:8000/a2a/litellm_agent` + +### Command-line helper + +Use the bundled script to drive hot-swaps and user messages over A2A: + +```bash +python task_agent/a2a_hot_swap.py \ + --url http://127.0.0.1:8000/a2a/litellm_agent \ + --model openai gpt-4o \ + --prompt "You are concise." \ + --config \ + --context demo-session +``` + +To send a follow-up prompt in the same session (with a larger timeout for long answers): + +```bash +python task_agent/a2a_hot_swap.py \ + --url http://127.0.0.1:8000/a2a/litellm_agent \ + --model openai gpt-4o \ + --prompt "You are concise." \ + --message "Give me a fuzzing harness." \ + --context demo-session \ + --timeout 120 +``` + +> Ensure the corresponding provider keys are present in `.env` (or passed via environment variables) before issuing model swaps. + +## Hot-Swap Tools + +The agent provides three special tools: + +### 1. `set_model` - Change the LLM Model + +Change the model during conversation: + +``` +User: Use the set_model tool to change to gpt-4o with openai provider +Agent: āœ… Model configured to: openai/gpt-4o + This change is active now! +``` + +**Parameters:** +- `model`: Model name (e.g., "gpt-4o", "claude-3-sonnet-20240229") +- `custom_llm_provider`: Optional provider prefix (e.g., "openai", "anthropic", "openrouter") + +**Examples:** +- OpenAI: `set_model(model="gpt-4o", custom_llm_provider="openai")` +- Anthropic: `set_model(model="claude-3-sonnet-20240229", custom_llm_provider="anthropic")` +- Google: `set_model(model="gemini-2.0-flash-001", custom_llm_provider="gemini")` + +### 2. `set_prompt` - Change System Prompt + +Update the system instructions: + +``` +User: Use set_prompt to change my behavior to "You are a helpful coding assistant" +Agent: āœ… System prompt updated: + You are a helpful coding assistant + + This change is active now! +``` + +### 3. `get_config` - View Configuration + +Check current model and prompt: + +``` +User: Use get_config to show me your configuration +Agent: šŸ“Š Current Configuration: + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” + Model: openai/gpt-4o + System Prompt: You are a helpful coding assistant + ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā” +``` + +## Testing + +### Basic A2A Client Test + +```bash +python agent/test_a2a_client.py +``` + +### Hot-Swap Functionality Test + +```bash +python agent/test_hotswap.py +``` + +This will: +1. Check initial configuration +2. Query with default model +3. Hot-swap to GPT-4o +4. Verify model changed +5. Change system prompt +6. Test new prompt behavior +7. Hot-swap to Claude +8. Verify final configuration + +### Command-Line Hot-Swap Helper + +You can trigger model and prompt changes directly against the A2A endpoint without the interactive CLI: + +```bash +# Start the agent first (in another terminal): +adk api_server --a2a --port 8000 task_agent + +# Apply swaps via pure A2A calls +python task_agent/a2a_hot_swap.py --model openai gpt-4o --prompt "You are concise." --config +python task_agent/a2a_hot_swap.py --model anthropic claude-3-sonnet-20240229 --context shared-session --config +python task_agent/a2a_hot_swap.py --prompt "" --context shared-session --config # Clear the prompt and show current state +``` + +`--model` accepts either `provider/model` or a provider/model pair. Add `--context` if you want to reuse the same conversation across invocations. Use `--config` to dump the agent's configuration after the changes are applied. + +## Supported Models + +### OpenAI +- `openai/gpt-4o` +- `openai/gpt-4-turbo` +- `openai/gpt-3.5-turbo` + +### Anthropic +- `anthropic/claude-3-opus-20240229` +- `anthropic/claude-3-sonnet-20240229` +- `anthropic/claude-3-haiku-20240307` + +### Google +- `gemini/gemini-2.0-flash-001` +- `gemini/gemini-2.5-pro-exp-03-25` +- `vertex_ai/gemini-2.0-flash-001` + +### OpenRouter +- `openrouter/anthropic/claude-3-opus` +- `openrouter/openai/gpt-4` +- Any model from OpenRouter catalog + +## How It Works + +### Session State +- Model and prompt settings are stored in session state +- Each session maintains its own configuration +- Settings persist across messages in the same session + +### Hot-Swap Mechanism +1. Tools update session state with new model/prompt +2. `before_agent_callback` checks for changes +3. If model changed, directly updates: `agent.model = LiteLlm(model=new_model)` +4. Dynamic instruction function reads custom prompt from session state + +### A2A Compatibility +- Agent card at `agent.json` defines A2A metadata +- Served at `/a2a/litellm_agent` endpoint +- Compatible with A2A client protocol + +## Example Usage + +### Interactive Session + +```python +from a2a.client import A2AClient +import asyncio + +async def chat(): + client = A2AClient("http://localhost:8000/a2a/litellm_agent") + context_id = "my-session-123" + + # Start with default model + async for msg in client.send_message("Hello!", context_id=context_id): + print(msg) + + # Switch to GPT-4 + async for msg in client.send_message( + "Use set_model with model gpt-4o and provider openai", + context_id=context_id + ): + print(msg) + + # Continue with new model + async for msg in client.send_message( + "Help me write a function", + context_id=context_id + ): + print(msg) + +asyncio.run(chat()) +``` + +## Troubleshooting + +### Model Not Found +- Ensure API key for the provider is set in `.env` +- Check model name is correct for the provider +- Verify LiteLLM supports the model (https://docs.litellm.ai/docs/providers) + +### Connection Refused +- Ensure the agent is running (`adk api_server --a2a task_agent`) +- Check the port matches (default: 8000) +- Verify no firewall blocking localhost + +### Hot-Swap Not Working +- Check that you're using the same `context_id` across messages +- Ensure the tool is being called (not just asked to switch) +- Look for `šŸ”„ Hot-swapped model to:` in server logs + +## Development + +### Adding New Tools + +```python +async def my_tool(tool_ctx: ToolContext, param: str) -> str: + """Your tool description.""" + # Access session state + tool_ctx.state["my_key"] = "my_value" + return "Tool result" + +# Add to agent +root_agent = LlmAgent( + # ... + tools=[set_model, set_prompt, get_config, my_tool], +) +``` + +### Modifying Callbacks + +```python +async def after_model_callback( + callback_context: CallbackContext, + llm_response: LlmResponse +) -> Optional[LlmResponse]: + """Modify response after model generates it.""" + # Your logic here + return llm_response +``` + +## License + +Apache 2.0 diff --git a/ai/agents/task_agent/__init__.py b/ai/agents/task_agent/__init__.py new file mode 100644 index 0000000..9d35e10 --- /dev/null +++ b/ai/agents/task_agent/__init__.py @@ -0,0 +1,5 @@ +"""Package entry point for the ADK-formatted hot swap agent.""" + +from .litellm_agent.agent import root_agent + +__all__ = ["root_agent"] diff --git a/ai/agents/task_agent/a2a_hot_swap.py b/ai/agents/task_agent/a2a_hot_swap.py new file mode 100644 index 0000000..8fbe140 --- /dev/null +++ b/ai/agents/task_agent/a2a_hot_swap.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Minimal A2A client utility for hot-swapping LiteLLM model/prompt.""" + +from __future__ import annotations + +import argparse +import asyncio +from typing import Optional +from uuid import uuid4 + +import httpx +from a2a.client import A2AClient +from a2a.client.errors import A2AClientHTTPError +from a2a.types import ( + JSONRPCErrorResponse, + Message, + MessageSendConfiguration, + MessageSendParams, + Part, + Role, + SendMessageRequest, + SendStreamingMessageRequest, + Task, + TaskArtifactUpdateEvent, + TaskStatusUpdateEvent, + TextPart, +) + +from litellm_agent.control import ( + HotSwapCommand, + build_control_message, + parse_model_spec, + serialize_model_spec, +) + +DEFAULT_URL = "http://localhost:8000/a2a/litellm_agent" + + +async def _collect_text(client: A2AClient, message: str, context_id: str) -> str: + """Send a message and collect streamed agent text into a single string.""" + + params = MessageSendParams( + configuration=MessageSendConfiguration(blocking=True), + message=Message( + context_id=context_id, + message_id=str(uuid4()), + role=Role.user, + parts=[Part(root=TextPart(text=message))], + ), + ) + + stream_request = SendStreamingMessageRequest(id=str(uuid4()), params=params) + buffer: list[str] = [] + try: + async for response in client.send_message_streaming(stream_request): + root = response.root + if isinstance(root, JSONRPCErrorResponse): + raise RuntimeError(f"A2A error: {root.error}") + + payload = root.result + buffer.extend(_extract_text(payload)) + except A2AClientHTTPError as exc: + if "text/event-stream" not in str(exc): + raise + + send_request = SendMessageRequest(id=str(uuid4()), params=params) + response = await client.send_message(send_request) + root = response.root + if isinstance(root, JSONRPCErrorResponse): + raise RuntimeError(f"A2A error: {root.error}") + payload = root.result + buffer.extend(_extract_text(payload)) + + if buffer: + buffer = list(dict.fromkeys(buffer)) + return "\n".join(buffer).strip() + + +def _extract_text( + result: Message | Task | TaskStatusUpdateEvent | TaskArtifactUpdateEvent, +) -> list[str]: + texts: list[str] = [] + if isinstance(result, Message): + if result.role is Role.agent: + for part in result.parts: + root_part = part.root + text = getattr(root_part, "text", None) + if text: + texts.append(text) + elif isinstance(result, Task) and result.history: + for msg in result.history: + if msg.role is Role.agent: + for part in msg.parts: + root_part = part.root + text = getattr(root_part, "text", None) + if text: + texts.append(text) + elif isinstance(result, TaskStatusUpdateEvent): + message = result.status.message + if message: + texts.extend(_extract_text(message)) + elif isinstance(result, TaskArtifactUpdateEvent): + artifact = result.artifact + if artifact and artifact.parts: + for part in artifact.parts: + root_part = part.root + text = getattr(root_part, "text", None) + if text: + texts.append(text) + return texts + + +def _split_model_args(model_args: Optional[list[str]]) -> tuple[Optional[str], Optional[str]]: + if not model_args: + return None, None + + if len(model_args) == 1: + return model_args[0], None + + provider = model_args[0] + model = " ".join(model_args[1:]) + return model, provider + + +async def hot_swap( + url: str, + *, + model_args: Optional[list[str]], + provider: Optional[str], + prompt: Optional[str], + message: Optional[str], + show_config: bool, + context_id: Optional[str], + timeout: float, +) -> None: + """Execute the requested hot-swap operations against the A2A endpoint.""" + + timeout_config = httpx.Timeout(timeout) + async with httpx.AsyncClient(timeout=timeout_config) as http_client: + client = A2AClient(url=url, httpx_client=http_client) + session_id = context_id or str(uuid4()) + + model, derived_provider = _split_model_args(model_args) + + if model: + spec = parse_model_spec(model, provider=provider or derived_provider) + payload = serialize_model_spec(spec) + control_msg = build_control_message(HotSwapCommand.MODEL, payload) + result = await _collect_text(client, control_msg, session_id) + print(f"Model response: {result or '(no response)'}") + + if prompt is not None: + control_msg = build_control_message(HotSwapCommand.PROMPT, prompt) + result = await _collect_text(client, control_msg, session_id) + print(f"Prompt response: {result or '(no response)'}") + + if show_config: + control_msg = build_control_message(HotSwapCommand.GET_CONFIG) + result = await _collect_text(client, control_msg, session_id) + print(f"Config:\n{result or '(no response)'}") + + if message: + result = await _collect_text(client, message, session_id) + print(f"Message response: {result or '(no response)'}") + + print(f"Context ID: {session_id}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--url", + default=DEFAULT_URL, + help=f"A2A endpoint for the agent (default: {DEFAULT_URL})", + ) + parser.add_argument( + "--model", + nargs="+", + help="LiteLLM model spec: either 'provider/model' or ' '.", + ) + parser.add_argument( + "--provider", + help="Optional LiteLLM provider when --model lacks a prefix.") + parser.add_argument( + "--prompt", + help="Set the system prompt (omit to leave unchanged; empty string clears it).", + ) + parser.add_argument( + "--message", + help="Send an additional user message after the swaps complete.") + parser.add_argument( + "--config", + action="store_true", + help="Print the agent configuration after performing swaps.") + parser.add_argument( + "--context", + help="Optional context/session identifier to reuse across calls.") + parser.add_argument( + "--timeout", + type=float, + default=60.0, + help="Request timeout (seconds) for A2A calls (default: 60).", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + asyncio.run( + hot_swap( + args.url, + model_args=args.model, + provider=args.provider, + prompt=args.prompt, + message=args.message, + show_config=args.config, + context_id=args.context, + timeout=args.timeout, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/ai/agents/task_agent/docker-compose.yml b/ai/agents/task_agent/docker-compose.yml new file mode 100644 index 0000000..b22a9ac --- /dev/null +++ b/ai/agents/task_agent/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.8' + +services: + task-agent: + build: + context: . + dockerfile: Dockerfile + container_name: fuzzforge-task-agent + ports: + - "10900:8000" + env_file: + - ../../../volumes/env/.env + environment: + - PORT=8000 + - PYTHONUNBUFFERED=1 + volumes: + # Mount volumes/env for runtime config access + - ../../../volumes/env:/app/config:ro + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 diff --git a/ai/agents/task_agent/litellm_agent/__init__.py b/ai/agents/task_agent/litellm_agent/__init__.py new file mode 100644 index 0000000..09c0772 --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/__init__.py @@ -0,0 +1,55 @@ +"""LiteLLM hot-swap agent package exports.""" + +from .agent import root_agent +from .callbacks import ( + before_agent_callback, + before_model_callback, + provide_instruction, +) +from .config import ( + AGENT_DESCRIPTION, + AGENT_NAME, + CONTROL_PREFIX, + DEFAULT_MODEL, + DEFAULT_PROVIDER, + STATE_MODEL_KEY, + STATE_PROVIDER_KEY, + STATE_PROMPT_KEY, +) +from .control import ( + HotSwapCommand, + ModelSpec, + build_control_message, + parse_control_message, + parse_model_spec, + serialize_model_spec, +) +from .state import HotSwapState, apply_state_to_agent +from .tools import HOTSWAP_TOOLS, get_config, set_model, set_prompt + +__all__ = [ + "root_agent", + "before_agent_callback", + "before_model_callback", + "provide_instruction", + "AGENT_DESCRIPTION", + "AGENT_NAME", + "CONTROL_PREFIX", + "DEFAULT_MODEL", + "DEFAULT_PROVIDER", + "STATE_MODEL_KEY", + "STATE_PROVIDER_KEY", + "STATE_PROMPT_KEY", + "HotSwapCommand", + "ModelSpec", + "HotSwapState", + "apply_state_to_agent", + "build_control_message", + "parse_control_message", + "parse_model_spec", + "serialize_model_spec", + "HOTSWAP_TOOLS", + "get_config", + "set_model", + "set_prompt", +] diff --git a/ai/agents/task_agent/litellm_agent/agent.json b/ai/agents/task_agent/litellm_agent/agent.json new file mode 100644 index 0000000..05f1112 --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/agent.json @@ -0,0 +1,24 @@ +{ + "name": "litellm_agent", + "description": "A flexible AI agent powered by LiteLLM with hot-swappable models from OpenRouter and other providers", + "url": "http://localhost:8000", + "version": "1.0.0", + "defaultInputModes": ["text/plain"], + "defaultOutputModes": ["text/plain"], + "capabilities": { + "streaming": true + }, + "skills": [ + { + "id": "litellm-general-purpose", + "name": "General Purpose AI Assistant", + "description": "A flexible AI assistant that can help with various tasks using any LiteLLM-supported model. Supports runtime model and prompt hot-swapping.", + "tags": ["ai", "assistant", "litellm", "flexible", "hot-swap"], + "examples": [ + "Help me write a Python function", + "Explain quantum computing", + "Switch to Claude model and help me code" + ] + } + ] +} diff --git a/ai/agents/task_agent/litellm_agent/agent.py b/ai/agents/task_agent/litellm_agent/agent.py new file mode 100644 index 0000000..c3189ce --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/agent.py @@ -0,0 +1,29 @@ +"""Root agent definition for the LiteLLM hot-swap shell.""" + +from __future__ import annotations + +from google.adk.agents import Agent + +from .callbacks import ( + before_agent_callback, + before_model_callback, + provide_instruction, +) +from .config import AGENT_DESCRIPTION, AGENT_NAME, DEFAULT_MODEL, DEFAULT_PROVIDER +from .state import HotSwapState +from .tools import HOTSWAP_TOOLS + +_initial_state = HotSwapState(model=DEFAULT_MODEL, provider=DEFAULT_PROVIDER) + +root_agent = Agent( + name=AGENT_NAME, + model=_initial_state.instantiate_llm(), + description=AGENT_DESCRIPTION, + instruction=provide_instruction, + tools=HOTSWAP_TOOLS, + before_agent_callback=before_agent_callback, + before_model_callback=before_model_callback, +) + + +__all__ = ["root_agent"] diff --git a/ai/agents/task_agent/litellm_agent/callbacks.py b/ai/agents/task_agent/litellm_agent/callbacks.py new file mode 100644 index 0000000..0faaa82 --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/callbacks.py @@ -0,0 +1,137 @@ +"""Callbacks and instruction providers for the LiteLLM hot-swap agent.""" + +from __future__ import annotations + +import logging +from typing import Optional + +from google.adk.agents.callback_context import CallbackContext +from google.adk.agents.readonly_context import ReadonlyContext +from google.adk.models.llm_request import LlmRequest +from google.genai import types + +from .config import CONTROL_PREFIX, DEFAULT_MODEL +from .control import HotSwapCommand, parse_control_message, parse_model_spec +from .prompts import BASE_INSTRUCTION +from .state import HotSwapState, apply_state_to_agent + +_LOGGER = logging.getLogger(__name__) + + +def provide_instruction(ctx: ReadonlyContext | None = None) -> str: + """Compose the system instruction using the stored state.""" + + state_mapping = getattr(ctx, "state", None) + state = HotSwapState.from_mapping(state_mapping) + prompt = state.prompt or BASE_INSTRUCTION + return f"{prompt}\n\nActive model: {state.display_model}" + + +def _ensure_state(callback_context: CallbackContext) -> HotSwapState: + state = HotSwapState.from_mapping(callback_context.state) + state.persist(callback_context.state) + return state + + +def _session_id(callback_context: CallbackContext) -> str: + session = getattr(callback_context, "session", None) + if session is None: + session = getattr(callback_context._invocation_context, "session", None) + return getattr(session, "id", "unknown-session") + + +async def before_model_callback( + callback_context: CallbackContext, + llm_request: LlmRequest, +) -> Optional[types.Content]: + """Ensure outgoing requests use the active model from session state.""" + + state = _ensure_state(callback_context) + try: + apply_state_to_agent(callback_context._invocation_context, state) + except Exception: # pragma: no cover - defensive logging + _LOGGER.exception( + "Failed to apply LiteLLM model '%s' (provider=%s) for session %s", + state.model, + state.provider, + callback_context.session.id, + ) + llm_request.model = state.model or DEFAULT_MODEL + return None + + +async def before_agent_callback( + callback_context: CallbackContext, +) -> Optional[types.Content]: + """Intercept hot-swap control messages and update session state.""" + + user_content = callback_context.user_content + if not user_content or not user_content.parts: + return None + + first_part = user_content.parts[0] + message_text = (first_part.text or "").strip() + if not message_text.startswith(CONTROL_PREFIX): + return None + + parsed = parse_control_message(message_text) + if not parsed: + return None + + command, payload = parsed + state = _ensure_state(callback_context) + + if command is HotSwapCommand.MODEL: + if not payload: + return _render("āŒ Missing model specification for hot-swap.") + try: + spec = parse_model_spec(payload) + except ValueError as exc: + return _render(f"āŒ Invalid model specification: {exc}") + + state.model = spec.model + state.provider = spec.provider + state.persist(callback_context.state) + try: + apply_state_to_agent(callback_context._invocation_context, state) + except Exception: # pragma: no cover - defensive logging + _LOGGER.exception( + "Failed to apply LiteLLM model '%s' (provider=%s) for session %s", + state.model, + state.provider, + _session_id(callback_context), + ) + _LOGGER.info( + "Hot-swapped model to %s (provider=%s, session=%s)", + state.model, + state.provider, + _session_id(callback_context), + ) + label = state.display_model + return _render(f"āœ… Model switched to: {label}") + + if command is HotSwapCommand.PROMPT: + prompt_value = (payload or "").strip() + state.prompt = prompt_value or None + state.persist(callback_context.state) + if state.prompt: + _LOGGER.info( + "Updated prompt for session %s", _session_id(callback_context) + ) + return _render( + "āœ… System prompt updated. This change takes effect immediately." + ) + return _render("āœ… System prompt cleared. Reverting to default instruction.") + + if command is HotSwapCommand.GET_CONFIG: + return _render(state.describe()) + + expected = ", ".join(HotSwapCommand.choices()) + return _render( + "āš ļø Unsupported hot-swap command. Available verbs: " + f"{expected}." + ) + + +def _render(message: str) -> types.ModelContent: + return types.ModelContent(parts=[types.Part(text=message)]) diff --git a/ai/agents/task_agent/litellm_agent/config.py b/ai/agents/task_agent/litellm_agent/config.py new file mode 100644 index 0000000..9b404bf --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/config.py @@ -0,0 +1,20 @@ +"""Configuration constants for the LiteLLM hot-swap agent.""" + +from __future__ import annotations + +import os + +AGENT_NAME = "litellm_agent" +AGENT_DESCRIPTION = ( + "A LiteLLM-backed shell that exposes hot-swappable model and prompt controls." +) + +DEFAULT_MODEL = os.getenv("LITELLM_MODEL", "gemini-2.0-flash-001") +DEFAULT_PROVIDER = os.getenv("LITELLM_PROVIDER") + +STATE_PREFIX = "app:litellm_agent/" +STATE_MODEL_KEY = f"{STATE_PREFIX}model" +STATE_PROVIDER_KEY = f"{STATE_PREFIX}provider" +STATE_PROMPT_KEY = f"{STATE_PREFIX}prompt" + +CONTROL_PREFIX = "[HOTSWAP" diff --git a/ai/agents/task_agent/litellm_agent/control.py b/ai/agents/task_agent/litellm_agent/control.py new file mode 100644 index 0000000..1c23379 --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/control.py @@ -0,0 +1,99 @@ +"""Control message helpers for hot-swapping model and prompt.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Tuple + +from .config import DEFAULT_PROVIDER + + +class HotSwapCommand(str, Enum): + """Supported control verbs embedded in user messages.""" + + MODEL = "MODEL" + PROMPT = "PROMPT" + GET_CONFIG = "GET_CONFIG" + + @classmethod + def choices(cls) -> tuple[str, ...]: + return tuple(item.value for item in cls) + + +@dataclass(frozen=True) +class ModelSpec: + """Represents a LiteLLM model and optional provider.""" + + model: str + provider: Optional[str] = None + + +_COMMAND_PATTERN = re.compile( + r"^\[HOTSWAP:(?P[A-Z_]+)(?::(?P.*))?\]$", +) + + +def parse_control_message(text: str) -> Optional[Tuple[HotSwapCommand, Optional[str]]]: + """Return hot-swap command tuple when the string matches the control format.""" + + match = _COMMAND_PATTERN.match(text.strip()) + if not match: + return None + + verb = match.group("verb") + if verb not in HotSwapCommand.choices(): + return None + + payload = match.group("payload") + return HotSwapCommand(verb), payload if payload else None + + +def build_control_message(command: HotSwapCommand, payload: Optional[str] = None) -> str: + """Serialise a control command for downstream clients.""" + + if command not in HotSwapCommand: + raise ValueError(f"Unsupported hot-swap command: {command}") + if payload is None or payload == "": + return f"[HOTSWAP:{command.value}]" + return f"[HOTSWAP:{command.value}:{payload}]" + + +def parse_model_spec(model: str, provider: Optional[str] = None) -> ModelSpec: + """Parse model/provider inputs into a structured ModelSpec.""" + + candidate = (model or "").strip() + if not candidate: + raise ValueError("Model name cannot be empty") + + if provider: + provider_clean = provider.strip() + if not provider_clean: + raise ValueError("Provider cannot be empty when supplied") + if "/" in candidate: + raise ValueError( + "Provide either provider/model or use provider argument, not both", + ) + return ModelSpec(model=candidate, provider=provider_clean) + + if "/" in candidate: + provider_part, model_part = candidate.split("/", 1) + provider_part = provider_part.strip() + model_part = model_part.strip() + if not provider_part or not model_part: + raise ValueError("Model spec must include provider and model when using '/' format") + return ModelSpec(model=model_part, provider=provider_part) + + if DEFAULT_PROVIDER: + return ModelSpec(model=candidate, provider=DEFAULT_PROVIDER.strip()) + + return ModelSpec(model=candidate, provider=None) + + +def serialize_model_spec(spec: ModelSpec) -> str: + """Render a ModelSpec to provider/model string for control messages.""" + + if spec.provider: + return f"{spec.provider}/{spec.model}" + return spec.model diff --git a/ai/agents/task_agent/litellm_agent/prompts.py b/ai/agents/task_agent/litellm_agent/prompts.py new file mode 100644 index 0000000..ec4d603 --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/prompts.py @@ -0,0 +1,9 @@ +"""System prompt templates for the LiteLLM agent.""" + +BASE_INSTRUCTION = ( + "You are a focused orchestration layer that relays between the user and a" + " LiteLLM managed model." + "\n- Keep answers concise and actionable." + "\n- Prefer plain language; reveal intermediate reasoning only when helpful." + "\n- Surface any tool results clearly with short explanations." +) diff --git a/ai/agents/task_agent/litellm_agent/state.py b/ai/agents/task_agent/litellm_agent/state.py new file mode 100644 index 0000000..460d961 --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/state.py @@ -0,0 +1,86 @@ +"""Session state utilities for the LiteLLM hot-swap agent.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping, MutableMapping, Optional + +from .config import ( + DEFAULT_MODEL, + DEFAULT_PROVIDER, + STATE_MODEL_KEY, + STATE_PROMPT_KEY, + STATE_PROVIDER_KEY, +) + + +@dataclass(slots=True) +class HotSwapState: + """Lightweight view of the hot-swap session state.""" + + model: str = DEFAULT_MODEL + provider: Optional[str] = None + prompt: Optional[str] = None + + @classmethod + def from_mapping(cls, mapping: Optional[Mapping[str, Any]]) -> "HotSwapState": + if not mapping: + return cls() + + raw_model = mapping.get(STATE_MODEL_KEY, DEFAULT_MODEL) + raw_provider = mapping.get(STATE_PROVIDER_KEY) + raw_prompt = mapping.get(STATE_PROMPT_KEY) + + model = raw_model.strip() if isinstance(raw_model, str) else DEFAULT_MODEL + provider = raw_provider.strip() if isinstance(raw_provider, str) else None + if not provider and DEFAULT_PROVIDER: + provider = DEFAULT_PROVIDER.strip() or None + prompt = raw_prompt.strip() if isinstance(raw_prompt, str) else None + return cls( + model=model or DEFAULT_MODEL, + provider=provider or None, + prompt=prompt or None, + ) + + def persist(self, store: MutableMapping[str, object]) -> None: + store[STATE_MODEL_KEY] = self.model + if self.provider: + store[STATE_PROVIDER_KEY] = self.provider + else: + store[STATE_PROVIDER_KEY] = None + store[STATE_PROMPT_KEY] = self.prompt + + def describe(self) -> str: + prompt_value = self.prompt if self.prompt else "(default prompt)" + provider_value = self.provider if self.provider else "(default provider)" + return ( + "šŸ“Š Current Configuration\n" + "ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”\n" + f"Model: {self.model}\n" + f"Provider: {provider_value}\n" + f"System Prompt: {prompt_value}\n" + "ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”ā”" + ) + + def instantiate_llm(self): + """Create a LiteLlm instance for the current state.""" + + from google.adk.models.lite_llm import LiteLlm # Lazy import to avoid cycle + + kwargs = {"model": self.model} + if self.provider: + kwargs["custom_llm_provider"] = self.provider + return LiteLlm(**kwargs) + + @property + def display_model(self) -> str: + if self.provider: + return f"{self.provider}/{self.model}" + return self.model + + +def apply_state_to_agent(invocation_context, state: HotSwapState) -> None: + """Update the provided agent with a LiteLLM instance matching state.""" + + agent = invocation_context.agent + agent.model = state.instantiate_llm() diff --git a/ai/agents/task_agent/litellm_agent/tools.py b/ai/agents/task_agent/litellm_agent/tools.py new file mode 100644 index 0000000..ff60a5f --- /dev/null +++ b/ai/agents/task_agent/litellm_agent/tools.py @@ -0,0 +1,64 @@ +"""Tool definitions exposed to the LiteLLM agent.""" + +from __future__ import annotations + +from typing import Optional + +from google.adk.tools import FunctionTool, ToolContext + +from .control import parse_model_spec +from .state import HotSwapState, apply_state_to_agent + + +async def set_model( + model: str, + *, + provider: Optional[str] = None, + tool_context: ToolContext, +) -> str: + """Hot-swap the active LiteLLM model for this session.""" + + spec = parse_model_spec(model, provider=provider) + state = HotSwapState.from_mapping(tool_context.state) + state.model = spec.model + state.provider = spec.provider + state.persist(tool_context.state) + try: + apply_state_to_agent(tool_context._invocation_context, state) + except Exception as exc: # pragma: no cover - defensive reporting + return f"āŒ Failed to apply model '{state.display_model}': {exc}" + return f"āœ… Model switched to: {state.display_model}" + + +async def set_prompt(prompt: str, *, tool_context: ToolContext) -> str: + """Update or clear the system prompt used for this session.""" + + state = HotSwapState.from_mapping(tool_context.state) + prompt_value = prompt.strip() + state.prompt = prompt_value or None + state.persist(tool_context.state) + if state.prompt: + return "āœ… System prompt updated. This change takes effect immediately." + return "āœ… System prompt cleared. Reverting to default instruction." + + +async def get_config(*, tool_context: ToolContext) -> str: + """Return a summary of the current model and prompt configuration.""" + + state = HotSwapState.from_mapping(tool_context.state) + return state.describe() + + +HOTSWAP_TOOLS = [ + FunctionTool(set_model), + FunctionTool(set_prompt), + FunctionTool(get_config), +] + + +__all__ = [ + "set_model", + "set_prompt", + "get_config", + "HOTSWAP_TOOLS", +] diff --git a/ai/agents/task_agent/main.py b/ai/agents/task_agent/main.py new file mode 100644 index 0000000..d675cad --- /dev/null +++ b/ai/agents/task_agent/main.py @@ -0,0 +1,13 @@ +"""ASGI entrypoint for containerized deployments.""" + +from pathlib import Path + +from google.adk.cli.fast_api import get_fast_api_app + +AGENT_DIR = Path(__file__).resolve().parent + +app = get_fast_api_app( + agents_dir=str(AGENT_DIR), + web=False, + a2a=True, +) diff --git a/ai/agents/task_agent/requirements.txt b/ai/agents/task_agent/requirements.txt new file mode 100644 index 0000000..132e57b --- /dev/null +++ b/ai/agents/task_agent/requirements.txt @@ -0,0 +1,4 @@ +google-adk +a2a-sdk[all] +litellm +python-dotenv diff --git a/ai/pyproject.toml b/ai/pyproject.toml index ef62383..d5c0e77 100644 --- a/ai/pyproject.toml +++ b/ai/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fuzzforge-ai" -version = "0.6.0" +version = "0.7.0" description = "FuzzForge AI orchestration module" readme = "README.md" requires-python = ">=3.11" diff --git a/ai/src/fuzzforge_ai/__init__.py b/ai/src/fuzzforge_ai/__init__.py index 5b343a2..cca81fc 100644 --- a/ai/src/fuzzforge_ai/__init__.py +++ b/ai/src/fuzzforge_ai/__init__.py @@ -3,6 +3,11 @@ FuzzForge AI Module - Agent-to-Agent orchestration system This module integrates the fuzzforge_ai components into FuzzForge, providing intelligent AI agent capabilities for security analysis. + +Usage: + from fuzzforge_ai.a2a_wrapper import send_agent_task + from fuzzforge_ai.agent import FuzzForgeAgent + from fuzzforge_ai.config_manager import ConfigManager """ # Copyright (c) 2025 FuzzingLabs # @@ -16,9 +21,4 @@ providing intelligent AI agent capabilities for security analysis. # Additional attribution and requirements are provided in the NOTICE file. -__version__ = "0.6.0" - -from .agent import FuzzForgeAgent -from .config_manager import ConfigManager - -__all__ = ['FuzzForgeAgent', 'ConfigManager'] \ No newline at end of file +__version__ = "0.6.0" \ No newline at end of file diff --git a/ai/src/fuzzforge_ai/__main__.py b/ai/src/fuzzforge_ai/__main__.py index 9a3e73b..297369f 100644 --- a/ai/src/fuzzforge_ai/__main__.py +++ b/ai/src/fuzzforge_ai/__main__.py @@ -1,3 +1,4 @@ +# ruff: noqa: E402 # Imports delayed for environment/logging setup """ FuzzForge A2A Server Run this to expose FuzzForge as an A2A-compatible agent @@ -78,7 +79,7 @@ def create_a2a_app(): print("\033[0m") # Reset color # Create A2A app - print(f"šŸš€ Starting FuzzForge A2A Server") + print("šŸš€ Starting FuzzForge A2A Server") print(f" Model: {fuzzforge.model}") if fuzzforge.cognee_url: print(f" Memory: Cognee at {fuzzforge.cognee_url}") @@ -86,7 +87,7 @@ def create_a2a_app(): app = create_custom_a2a_app(fuzzforge.adk_agent, port=port, executor=fuzzforge.executor) - print(f"\nāœ… FuzzForge A2A Server ready!") + print("\nāœ… FuzzForge A2A Server ready!") print(f" Agent card: http://localhost:{port}/.well-known/agent-card.json") print(f" A2A endpoint: http://localhost:{port}/") print(f"\nšŸ“” Other agents can register FuzzForge at: http://localhost:{port}") @@ -101,7 +102,7 @@ def main(): app = create_a2a_app() port = int(os.getenv('FUZZFORGE_PORT', 10100)) - print(f"\nšŸŽÆ Starting server with uvicorn...") + print("\nšŸŽÆ Starting server with uvicorn...") uvicorn.run(app, host="127.0.0.1", port=port) diff --git a/ai/src/fuzzforge_ai/a2a_server.py b/ai/src/fuzzforge_ai/a2a_server.py index 310451c..8c67e8e 100644 --- a/ai/src/fuzzforge_ai/a2a_server.py +++ b/ai/src/fuzzforge_ai/a2a_server.py @@ -18,7 +18,6 @@ from typing import Optional, Union from starlette.applications import Starlette from starlette.responses import Response, FileResponse -from starlette.routing import Route from google.adk.a2a.executor.a2a_agent_executor import A2aAgentExecutor from google.adk.a2a.utils.agent_card_builder import AgentCardBuilder diff --git a/ai/src/fuzzforge_ai/a2a_wrapper.py b/ai/src/fuzzforge_ai/a2a_wrapper.py new file mode 100644 index 0000000..0535404 --- /dev/null +++ b/ai/src/fuzzforge_ai/a2a_wrapper.py @@ -0,0 +1,288 @@ +""" +A2A Wrapper Module for FuzzForge +Programmatic interface to send tasks to A2A agents with custom model/prompt/context +""" +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +from __future__ import annotations + +from typing import Optional, Any +from uuid import uuid4 + +import httpx +from a2a.client import A2AClient +from a2a.client.errors import A2AClientHTTPError +from a2a.types import ( + JSONRPCErrorResponse, + Message, + MessageSendConfiguration, + MessageSendParams, + Part, + Role, + SendMessageRequest, + SendStreamingMessageRequest, + Task, + TaskArtifactUpdateEvent, + TaskStatusUpdateEvent, + TextPart, +) + + +class A2ATaskResult: + """Result from an A2A agent task""" + + def __init__(self, text: str, context_id: str, raw_response: Any = None): + self.text = text + self.context_id = context_id + self.raw_response = raw_response + + def __str__(self) -> str: + return self.text + + def __repr__(self) -> str: + return f"A2ATaskResult(text={self.text[:50]}..., context_id={self.context_id})" + + +def _build_control_message(command: str, payload: Optional[str] = None) -> str: + """Build a control message for hot-swapping agent configuration""" + if payload is None or payload == "": + return f"[HOTSWAP:{command}]" + return f"[HOTSWAP:{command}:{payload}]" + + +def _extract_text( + result: Message | Task | TaskStatusUpdateEvent | TaskArtifactUpdateEvent, +) -> list[str]: + """Extract text content from A2A response objects""" + texts: list[str] = [] + if isinstance(result, Message): + if result.role is Role.agent: + for part in result.parts: + root_part = part.root + text = getattr(root_part, "text", None) + if text: + texts.append(text) + elif isinstance(result, Task) and result.history: + for msg in result.history: + if msg.role is Role.agent: + for part in msg.parts: + root_part = part.root + text = getattr(root_part, "text", None) + if text: + texts.append(text) + elif isinstance(result, TaskStatusUpdateEvent): + message = result.status.message + if message: + texts.extend(_extract_text(message)) + elif isinstance(result, TaskArtifactUpdateEvent): + artifact = result.artifact + if artifact and artifact.parts: + for part in artifact.parts: + root_part = part.root + text = getattr(root_part, "text", None) + if text: + texts.append(text) + return texts + + +async def _send_message( + client: A2AClient, + message: str, + context_id: str, +) -> str: + """Send a message to the A2A agent and collect the response""" + + params = MessageSendParams( + configuration=MessageSendConfiguration(blocking=True), + message=Message( + context_id=context_id, + message_id=str(uuid4()), + role=Role.user, + parts=[Part(root=TextPart(text=message))], + ), + ) + + stream_request = SendStreamingMessageRequest(id=str(uuid4()), params=params) + buffer: list[str] = [] + + try: + async for response in client.send_message_streaming(stream_request): + root = response.root + if isinstance(root, JSONRPCErrorResponse): + raise RuntimeError(f"A2A error: {root.error}") + + payload = root.result + buffer.extend(_extract_text(payload)) + except A2AClientHTTPError as exc: + if "text/event-stream" not in str(exc): + raise + + # Fallback to non-streaming + send_request = SendMessageRequest(id=str(uuid4()), params=params) + response = await client.send_message(send_request) + root = response.root + if isinstance(root, JSONRPCErrorResponse): + raise RuntimeError(f"A2A error: {root.error}") + payload = root.result + buffer.extend(_extract_text(payload)) + + if buffer: + buffer = list(dict.fromkeys(buffer)) # Remove duplicates + return "\n".join(buffer).strip() + + +async def send_agent_task( + url: str, + message: str, + *, + model: Optional[str] = None, + provider: Optional[str] = None, + prompt: Optional[str] = None, + context: Optional[str] = None, + timeout: float = 120.0, +) -> A2ATaskResult: + """ + Send a task to an A2A agent with optional model/prompt configuration. + + Args: + url: A2A endpoint URL (e.g., "http://127.0.0.1:8000/a2a/litellm_agent") + message: The task message to send to the agent + model: Optional model name (e.g., "gpt-4o", "gemini-2.0-flash") + provider: Optional provider name (e.g., "openai", "gemini") + prompt: Optional system prompt to set before sending the message + context: Optional context/session ID (generated if not provided) + timeout: Request timeout in seconds (default: 120) + + Returns: + A2ATaskResult with the agent's response text and context ID + + Example: + >>> result = await send_agent_task( + ... url="http://127.0.0.1:8000/a2a/litellm_agent", + ... model="gpt-4o", + ... provider="openai", + ... prompt="You are concise.", + ... message="Give me a fuzzing harness.", + ... context="fuzzing", + ... timeout=120 + ... ) + >>> print(result.text) + """ + timeout_config = httpx.Timeout(timeout) + context_id = context or str(uuid4()) + + async with httpx.AsyncClient(timeout=timeout_config) as http_client: + client = A2AClient(url=url, httpx_client=http_client) + + # Set model if provided + if model: + model_spec = f"{provider}/{model}" if provider else model + control_msg = _build_control_message("MODEL", model_spec) + await _send_message(client, control_msg, context_id) + + # Set prompt if provided + if prompt is not None: + control_msg = _build_control_message("PROMPT", prompt) + await _send_message(client, control_msg, context_id) + + # Send the actual task message + response_text = await _send_message(client, message, context_id) + + return A2ATaskResult( + text=response_text, + context_id=context_id, + ) + + +async def get_agent_config( + url: str, + context: Optional[str] = None, + timeout: float = 60.0, +) -> str: + """ + Get the current configuration of an A2A agent. + + Args: + url: A2A endpoint URL + context: Optional context/session ID + timeout: Request timeout in seconds + + Returns: + Configuration string from the agent + """ + timeout_config = httpx.Timeout(timeout) + context_id = context or str(uuid4()) + + async with httpx.AsyncClient(timeout=timeout_config) as http_client: + client = A2AClient(url=url, httpx_client=http_client) + control_msg = _build_control_message("GET_CONFIG") + config_text = await _send_message(client, control_msg, context_id) + return config_text + + +async def hot_swap_model( + url: str, + model: str, + provider: Optional[str] = None, + context: Optional[str] = None, + timeout: float = 60.0, +) -> str: + """ + Hot-swap the model of an A2A agent without sending a task. + + Args: + url: A2A endpoint URL + model: Model name to switch to + provider: Optional provider name + context: Optional context/session ID + timeout: Request timeout in seconds + + Returns: + Response from the agent + """ + timeout_config = httpx.Timeout(timeout) + context_id = context or str(uuid4()) + + async with httpx.AsyncClient(timeout=timeout_config) as http_client: + client = A2AClient(url=url, httpx_client=http_client) + model_spec = f"{provider}/{model}" if provider else model + control_msg = _build_control_message("MODEL", model_spec) + response = await _send_message(client, control_msg, context_id) + return response + + +async def hot_swap_prompt( + url: str, + prompt: str, + context: Optional[str] = None, + timeout: float = 60.0, +) -> str: + """ + Hot-swap the system prompt of an A2A agent. + + Args: + url: A2A endpoint URL + prompt: System prompt to set + context: Optional context/session ID + timeout: Request timeout in seconds + + Returns: + Response from the agent + """ + timeout_config = httpx.Timeout(timeout) + context_id = context or str(uuid4()) + + async with httpx.AsyncClient(timeout=timeout_config) as http_client: + client = A2AClient(url=url, httpx_client=http_client) + control_msg = _build_control_message("PROMPT", prompt) + response = await _send_message(client, control_msg, context_id) + return response diff --git a/ai/src/fuzzforge_ai/agent.py b/ai/src/fuzzforge_ai/agent.py index 0cedc7a..b33b6cd 100644 --- a/ai/src/fuzzforge_ai/agent.py +++ b/ai/src/fuzzforge_ai/agent.py @@ -60,7 +60,7 @@ class FuzzForgeAgent: debug=os.getenv('FUZZFORGE_DEBUG', '0') == '1', memory_service=self.memory_service, session_persistence=os.getenv('SESSION_PERSISTENCE', 'inmemory'), - fuzzforge_mcp_url=os.getenv('FUZZFORGE_MCP_URL'), + fuzzforge_mcp_url=None, # Disabled ) # Create Hybrid Memory Manager (ADK + Cognee direct integration) diff --git a/ai/src/fuzzforge_ai/agent_card.py b/ai/src/fuzzforge_ai/agent_card.py index 9150092..175473d 100644 --- a/ai/src/fuzzforge_ai/agent_card.py +++ b/ai/src/fuzzforge_ai/agent_card.py @@ -15,7 +15,7 @@ Defines what FuzzForge can do and how others can discover it from dataclasses import dataclass -from typing import List, Optional, Dict, Any +from typing import List, Dict, Any @dataclass class AgentSkill: @@ -172,7 +172,6 @@ def get_fuzzforge_agent_card(url: str = "http://localhost:10100") -> AgentCard: orchestration_skill, memory_skill, conversation_skill, - workflow_automation_skill, agent_management_skill ], capabilities=fuzzforge_capabilities, diff --git a/ai/src/fuzzforge_ai/agent_executor.py b/ai/src/fuzzforge_ai/agent_executor.py index 6c0be70..fd1f1d9 100644 --- a/ai/src/fuzzforge_ai/agent_executor.py +++ b/ai/src/fuzzforge_ai/agent_executor.py @@ -1,3 +1,4 @@ +# ruff: noqa: E402 # Imports delayed for environment/logging setup """FuzzForge Agent Executor - orchestrates workflows and delegation.""" # Copyright (c) 2025 FuzzingLabs # @@ -12,7 +13,6 @@ import asyncio -import base64 import time import uuid import json @@ -174,7 +174,7 @@ class FuzzForgeExecutor: else: # Run now if no loop is running loop.run_until_complete(self._register_agent_async(url, name)) - except: + except Exception: # Ignore auto-registration failures pass except Exception as e: @@ -392,7 +392,7 @@ class FuzzForgeExecutor: user_email = f"project_{config.get_project_context()['project_id']}@fuzzforge.example" user = await get_user(user_email) cognee.set_user(user) - except Exception as e: + except Exception: pass # User context not critical # Use cognee search directly for maximum flexibility @@ -452,11 +452,11 @@ class FuzzForgeExecutor: try: user = await get_user(user_email) logger.info(f"Using existing user: {user_email}") - except: + except Exception: try: user = await create_user(user_email, user_tenant) logger.info(f"Created new user: {user_email}") - except: + except Exception: user = None if user: @@ -583,7 +583,6 @@ class FuzzForgeExecutor: pattern: Glob pattern (e.g. '*.py', '**/*.js', '') """ try: - from pathlib import Path # Get project root from config config = ProjectConfigManager() @@ -648,7 +647,6 @@ class FuzzForgeExecutor: max_lines: Maximum lines to read (0 for all, default 200 for large files) """ try: - from pathlib import Path # Get project root from config config = ProjectConfigManager() @@ -711,7 +709,6 @@ class FuzzForgeExecutor: """ try: import re - from pathlib import Path # Get project root from config config = ProjectConfigManager() @@ -757,7 +754,7 @@ class FuzzForgeExecutor: result = f"Found '{search_pattern}' in {len(matches)} locations (searched {files_searched} files):\n" result += "\n".join(matches[:50]) if len(matches) >= 50: - result += f"\n... (showing first 50 matches)" + result += "\n... (showing first 50 matches)" return result else: return f"No matches found for '{search_pattern}' in {files_searched} files matching '{file_pattern}'" @@ -845,15 +842,15 @@ class FuzzForgeExecutor: elif normalised_mode in {"read_write", "readwrite", "rw"}: normalised_mode = "rw" else: - # Fall back to Prefect defaults if we can't recognise the input + # Fall back to read-only if we can't recognise the input normalised_mode = "ro" - # Resolve the target path to an absolute path for Prefect's validation + # Resolve the target path to an absolute path for validation resolved_path = target_path or "." try: resolved_path = str(Path(resolved_path).expanduser().resolve()) except Exception: - # If resolution fails, Prefect will surface the validation error ā€“ use the raw value + # If resolution fails, use the raw value resolved_path = target_path # Ensure configuration objects default to dictionaries instead of None @@ -1088,7 +1085,7 @@ class FuzzForgeExecutor: def _build_instruction(self) -> str: """Build the agent's instruction prompt""" - instruction = f"""You are FuzzForge, an intelligent A2A orchestrator with dual memory systems. + instruction = """You are FuzzForge, an intelligent A2A orchestrator with dual memory systems. ## Your Core Responsibilities: @@ -1708,7 +1705,7 @@ Be concise and intelligent in your responses.""" if self.agentops_trace: try: agentops.end_trace() - except: + except Exception: pass # Cancel background monitors diff --git a/ai/src/fuzzforge_ai/cli.py b/ai/src/fuzzforge_ai/cli.py index b63f7bd..4f5549f 100755 --- a/ai/src/fuzzforge_ai/cli.py +++ b/ai/src/fuzzforge_ai/cli.py @@ -1,3 +1,4 @@ +# ruff: noqa: E402 # Imports delayed for environment/logging setup #!/usr/bin/env python3 # Copyright (c) 2025 FuzzingLabs # @@ -26,7 +27,6 @@ import random from datetime import datetime from contextlib import contextmanager from pathlib import Path -from typing import Any from dotenv import load_dotenv @@ -90,18 +90,12 @@ except ImportError: from rich.console import Console from rich.table import Table from rich.panel import Panel -from rich.prompt import Prompt from rich import box -from google.adk.events.event import Event -from google.adk.events.event_actions import EventActions -from google.genai import types as gen_types from .agent import FuzzForgeAgent -from .agent_card import get_fuzzforge_agent_card from .config_manager import ConfigManager from .config_bridge import ProjectConfigManager -from .remote_agent import RemoteAgentConnection console = Console() @@ -243,7 +237,7 @@ class FuzzForgeCLI: ) ) if self.agent.executor.agentops_trace: - console.print(f"Tracking: [medium_purple1]AgentOps active[/medium_purple1]") + console.print("Tracking: [medium_purple1]AgentOps active[/medium_purple1]") # Show skills console.print("\nSkills:") @@ -320,7 +314,7 @@ class FuzzForgeCLI: url=args.strip(), description=description ) - console.print(f" [dim]Saved to config for auto-registration[/dim]") + console.print(" [dim]Saved to config for auto-registration[/dim]") else: console.print(f"[red]Failed: {result['error']}[/red]") @@ -346,9 +340,9 @@ class FuzzForgeCLI: # Remove from config if self.config_manager.remove_registered_agent(name=agent_to_remove['name'], url=agent_to_remove['url']): console.print(f"āœ… Unregistered: [bold]{agent_to_remove['name']}[/bold]") - console.print(f" [dim]Removed from config (won't auto-register next time)[/dim]") + console.print(" [dim]Removed from config (won't auto-register next time)[/dim]") else: - console.print(f"[yellow]Agent unregistered from session but not found in config[/yellow]") + console.print("[yellow]Agent unregistered from session but not found in config[/yellow]") async def cmd_list(self, args: str = "") -> None: """List registered agents""" @@ -435,7 +429,7 @@ class FuzzForgeCLI: text = data['parts'][0].get('text', '')[:150] role = data.get('role', 'unknown') console.print(f"{i}. [{role}]: {text}...") - except: + except Exception: console.print(f"{i}. {content[:150]}...") else: console.print("[yellow]No matches found in SQLite either[/yellow]") @@ -699,7 +693,7 @@ class FuzzForgeCLI: ) console.print(table) - console.print(f"\n[dim]Use /artifacts to view artifact content[/dim]") + console.print("\n[dim]Use /artifacts to view artifact content[/dim]") async def cmd_tasks(self, args: str = "") -> None: """List tasks or show details for a specific task.""" diff --git a/ai/src/fuzzforge_ai/cognee_integration.py b/ai/src/fuzzforge_ai/cognee_integration.py index 2f134ce..90d005d 100644 --- a/ai/src/fuzzforge_ai/cognee_integration.py +++ b/ai/src/fuzzforge_ai/cognee_integration.py @@ -16,9 +16,7 @@ Can be reused by external agents and other components import os -import asyncio -import json -from typing import Dict, List, Any, Optional, Union +from typing import Dict, Any, Optional from pathlib import Path @@ -189,33 +187,69 @@ class CogneeProjectIntegration: except Exception as e: return {"error": f"Failed to list data: {e}"} + async def cognify_text(self, text: str, dataset: str = None) -> Dict[str, Any]: + """ + Cognify text content into knowledge graph + + Args: + text: Text to cognify + dataset: Dataset name (defaults to project_name_codebase) + + Returns: + Dict containing cognify results + """ + if not self._initialized: + await self.initialize() + + if not self._initialized: + return {"error": "Cognee not initialized"} + + if not dataset: + dataset = f"{self.project_context['project_name']}_codebase" + + try: + # Add text to dataset + await self._cognee.add([text], dataset_name=dataset) + + # Process (cognify) the dataset + await self._cognee.cognify([dataset]) + + return { + "text_length": len(text), + "dataset": dataset, + "project": self.project_context["project_name"], + "status": "success" + } + except Exception as e: + return {"error": f"Cognify failed: {e}"} + async def ingest_text_to_dataset(self, text: str, dataset: str = None) -> Dict[str, Any]: """ Ingest text content into a specific dataset - + Args: text: Text to ingest dataset: Dataset name (defaults to project_name_codebase) - + Returns: Dict containing ingest results """ if not self._initialized: await self.initialize() - + if not self._initialized: return {"error": "Cognee not initialized"} - + if not dataset: dataset = f"{self.project_context['project_name']}_codebase" - + try: # Add text to dataset await self._cognee.add([text], dataset_name=dataset) - + # Process (cognify) the dataset await self._cognee.cognify([dataset]) - + return { "text_length": len(text), "dataset": dataset, diff --git a/ai/src/fuzzforge_ai/cognee_service.py b/ai/src/fuzzforge_ai/cognee_service.py index dea5d5d..968e956 100644 --- a/ai/src/fuzzforge_ai/cognee_service.py +++ b/ai/src/fuzzforge_ai/cognee_service.py @@ -15,11 +15,9 @@ Provides integrated Cognee functionality for codebase analysis and knowledge gra import os -import asyncio import logging from pathlib import Path -from typing import Dict, List, Any, Optional -from datetime import datetime +from typing import Dict, List, Any logger = logging.getLogger(__name__) @@ -158,7 +156,7 @@ class CogneeService: self._user = await get_user(fallback_email) logger.info(f"Using existing user: {fallback_email}") return - except: + except Exception: # User doesn't exist, try to create fallback pass diff --git a/ai/src/fuzzforge_ai/config_bridge.py b/ai/src/fuzzforge_ai/config_bridge.py index 668f607..32a7905 100644 --- a/ai/src/fuzzforge_ai/config_bridge.py +++ b/ai/src/fuzzforge_ai/config_bridge.py @@ -13,7 +13,7 @@ try: from fuzzforge_cli.config import ProjectConfigManager as _ProjectConfigManager -except ImportError as exc: # pragma: no cover - used when CLI not available +except ImportError: # pragma: no cover - used when CLI not available class _ProjectConfigManager: # type: ignore[no-redef] """Fallback implementation that raises a helpful error.""" @@ -21,10 +21,10 @@ except ImportError as exc: # pragma: no cover - used when CLI not available raise ImportError( "ProjectConfigManager is unavailable. Install the FuzzForge CLI " "package or supply a compatible configuration object." - ) from exc + ) def __getattr__(name): # pragma: no cover - defensive - raise ImportError("ProjectConfigManager unavailable") from exc + raise ImportError("ProjectConfigManager unavailable") ProjectConfigManager = _ProjectConfigManager diff --git a/ai/src/fuzzforge_ai/memory_service.py b/ai/src/fuzzforge_ai/memory_service.py index 8f2446d..f00b7c3 100644 --- a/ai/src/fuzzforge_ai/memory_service.py +++ b/ai/src/fuzzforge_ai/memory_service.py @@ -16,15 +16,12 @@ Separate from Cognee which will be used for RAG/codebase analysis import os -import json -from typing import Dict, List, Any, Optional -from datetime import datetime +from typing import Dict, Any import logging # ADK Memory imports from google.adk.memory import InMemoryMemoryService, BaseMemoryService from google.adk.memory.base_memory_service import SearchMemoryResponse -from google.adk.memory.memory_entry import MemoryEntry # Optional VertexAI Memory Bank try: diff --git a/ai/src/fuzzforge_ai/remote_agent.py b/ai/src/fuzzforge_ai/remote_agent.py index 52da844..bac6872 100644 --- a/ai/src/fuzzforge_ai/remote_agent.py +++ b/ai/src/fuzzforge_ai/remote_agent.py @@ -37,7 +37,7 @@ class RemoteAgentConnection: response.raise_for_status() self.agent_card = response.json() return self.agent_card - except: + except Exception: # Try old path for compatibility try: response = await self.client.get(f"{self.url}/.well-known/agent.json") diff --git a/backend/Dockerfile b/backend/Dockerfile index e72c50c..7a49c84 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -17,25 +17,21 @@ RUN apt-get update && apt-get install -y \ # Docker client configuration removed - localhost:5001 doesn't require insecure registry config -# Install uv for faster package management -RUN pip install uv - # Copy project files COPY pyproject.toml ./ -COPY uv.lock ./ -# Install dependencies -RUN uv sync --no-dev +# Install dependencies with pip +RUN pip install --no-cache-dir -e . # Copy source code COPY . . -# Expose port -EXPOSE 8000 +# Expose ports (API on 8000, MCP on 8010) +EXPOSE 8000 8010 # Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD curl -f http://localhost:8000/health || exit 1 # Start the application -CMD ["uv", "run", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/backend/README.md b/backend/README.md index 54c7004..74589b0 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # FuzzForge Backend -A stateless API server for security testing workflow orchestration using Prefect. This system dynamically discovers workflows, executes them in isolated Docker containers with volume mounting, and returns findings in SARIF format. +A stateless API server for security testing workflow orchestration using Temporal. This system dynamically discovers workflows, executes them in isolated worker environments, and returns findings in SARIF format. ## Architecture Overview @@ -8,17 +8,17 @@ A stateless API server for security testing workflow orchestration using Prefect 1. **Workflow Discovery System**: Automatically discovers workflows at startup 2. **Module System**: Reusable components (scanner, analyzer, reporter) with a common interface -3. **Prefect Integration**: Handles container orchestration, workflow execution, and monitoring -4. **Volume Mounting**: Secure file access with configurable permissions (ro/rw) +3. **Temporal Integration**: Handles workflow orchestration, execution, and monitoring with vertical workers +4. **File Upload & Storage**: HTTP multipart upload to MinIO for target files 5. **SARIF Output**: Standardized security findings format ### Key Features - **Stateless**: No persistent data, fully scalable - **Generic**: No hardcoded workflows, automatic discovery -- **Isolated**: Each workflow runs in its own Docker container +- **Isolated**: Each workflow runs in specialized vertical workers - **Extensible**: Easy to add new workflows and modules -- **Secure**: Read-only volume mounts by default, path validation +- **Secure**: File upload with MinIO storage, automatic cleanup via lifecycle policies - **Observable**: Comprehensive logging and status tracking ## Quick Start @@ -32,19 +32,17 @@ A stateless API server for security testing workflow orchestration using Prefect From the project root, start all services: ```bash -docker-compose up -d +docker-compose -f docker-compose.temporal.yaml up -d ``` This will start: -- Prefect server (API at http://localhost:4200/api) -- PostgreSQL database -- Redis cache -- Docker registry (port 5001) -- Prefect worker (for running workflows) +- Temporal server (Web UI at http://localhost:8233, gRPC at :7233) +- MinIO (S3 storage at http://localhost:9000, Console at http://localhost:9001) +- PostgreSQL database (for Temporal state) +- Vertical workers (worker-rust, worker-android, worker-web, etc.) - FuzzForge backend API (port 8000) -- FuzzForge MCP server (port 8010) -**Note**: The Prefect UI at http://localhost:4200 is not currently accessible from the host due to the API being configured for inter-container communication. Use the REST API or MCP interface instead. +**Note**: MinIO console login: `fuzzforge` / `fuzzforge123` ## API Endpoints @@ -54,7 +52,8 @@ This will start: - `GET /workflows/{name}/metadata` - Get workflow metadata and parameters - `GET /workflows/{name}/parameters` - Get workflow parameter schema - `GET /workflows/metadata/schema` - Get metadata.yaml schema -- `POST /workflows/{name}/submit` - Submit a workflow for execution +- `POST /workflows/{name}/submit` - Submit a workflow for execution (path-based, legacy) +- `POST /workflows/{name}/upload-and-submit` - **Upload local files and submit workflow** (recommended) ### Runs @@ -68,12 +67,13 @@ Each workflow must have: ``` toolbox/workflows/{workflow_name}/ - workflow.py # Prefect flow definition - metadata.yaml # Mandatory metadata (parameters, version, etc.) - Dockerfile # Optional custom container definition - requirements.txt # Optional Python dependencies + workflow.py # Temporal workflow definition + metadata.yaml # Mandatory metadata (parameters, version, vertical, etc.) + requirements.txt # Optional Python dependencies (installed in vertical worker) ``` +**Note**: With Temporal architecture, workflows run in pre-built vertical workers (e.g., `worker-rust`, `worker-android`), not individual Docker containers. The workflow code is mounted as a volume and discovered at runtime. + ### Example metadata.yaml ```yaml @@ -82,15 +82,12 @@ version: "1.0.0" description: "Comprehensive security analysis workflow" author: "FuzzForge Team" category: "comprehensive" +vertical: "rust" # Routes to worker-rust tags: - "security" - "analysis" - "comprehensive" -supported_volume_modes: - - "ro" - - "rw" - requirements: tools: - "file_scanner" @@ -110,11 +107,6 @@ parameters: type: string default: "/workspace" description: "Path to analyze" - volume_mode: - type: string - enum: ["ro", "rw"] - default: "ro" - description: "Volume mount mode" scanner_config: type: object description: "Scanner configuration" @@ -159,7 +151,6 @@ curl -X POST "http://localhost:8000/workflows/security_assessment/submit" \ -H "Content-Type: application/json" \ -d '{ "target_path": "/tmp/project", - "volume_mode": "ro", "resource_limits": { "memory_limit": "1Gi", "cpu_limit": "1" @@ -169,6 +160,54 @@ curl -X POST "http://localhost:8000/workflows/security_assessment/submit" \ Resource precedence: User limits > Workflow requirements > System defaults +## File Upload and Target Access + +### Upload Endpoint + +The backend provides an upload endpoint for submitting workflows with local files: + +``` +POST /workflows/{workflow_name}/upload-and-submit +Content-Type: multipart/form-data + +Parameters: + file: File upload (supports .tar.gz for directories) + parameters: JSON string of workflow parameters (optional) + timeout: Execution timeout in seconds (optional) +``` + +Example using curl: + +```bash +# Upload a directory (create tarball first) +tar -czf project.tar.gz /path/to/project +curl -X POST "http://localhost:8000/workflows/security_assessment/upload-and-submit" \ + -F "file=@project.tar.gz" \ + -F "parameters={\"check_secrets\":true}" + +# Upload a single file +curl -X POST "http://localhost:8000/workflows/security_assessment/upload-and-submit" \ + -F "file=@binary.elf" +``` + +### Storage Flow + +1. **CLI/API uploads file** via HTTP multipart +2. **Backend receives file** and streams to temporary location (max 10GB) +3. **Backend uploads to MinIO** with generated `target_id` +4. **Workflow is submitted** to Temporal with `target_id` +5. **Worker downloads target** from MinIO to local cache +6. **Workflow processes target** from cache +7. **MinIO lifecycle policy** deletes files after 7 days + +### Advantages + +- **No host filesystem access required** - workers can run anywhere +- **Automatic cleanup** - lifecycle policies prevent disk exhaustion +- **Caching** - repeated workflows reuse cached targets +- **Multi-host ready** - targets accessible from any worker +- **Secure** - isolated storage, no arbitrary host path access + ## Module Development Modules implement the `BaseModule` interface: @@ -198,12 +237,24 @@ class MyModule(BaseModule): ## Submitting a Workflow +### With File Upload (Recommended) + ```bash +# Automatic tarball and upload +tar -czf project.tar.gz /home/user/project +curl -X POST "http://localhost:8000/workflows/security_assessment/upload-and-submit" \ + -F "file=@project.tar.gz" \ + -F "parameters={\"scanner_config\":{\"patterns\":[\"*.py\"]},\"analyzer_config\":{\"check_secrets\":true}}" +``` + +### Legacy Path-Based Submission + +```bash +# Only works if backend and target are on same machine curl -X POST "http://localhost:8000/workflows/security_assessment/submit" \ -H "Content-Type: application/json" \ -d '{ "target_path": "/home/user/project", - "volume_mode": "ro", "parameters": { "scanner_config": {"patterns": ["*.py"]}, "analyzer_config": {"check_secrets": true} @@ -235,23 +286,31 @@ Returns SARIF-formatted findings: ## Security Considerations -1. **Volume Mounting**: Only allowed directories can be mounted -2. **Read-Only Default**: Volumes mounted as read-only unless explicitly set -3. **Container Isolation**: Each workflow runs in an isolated container -4. **Resource Limits**: Can set CPU/memory limits via Prefect -5. **Network Isolation**: Containers use bridge networking +1. **File Upload Security**: Files uploaded to MinIO with isolated storage +2. **Read-Only Default**: Target files accessed as read-only unless explicitly set +3. **Worker Isolation**: Each workflow runs in isolated vertical workers +4. **Resource Limits**: Can set CPU/memory limits per worker +5. **Automatic Cleanup**: MinIO lifecycle policies delete old files after 7 days ## Development ### Adding a New Workflow 1. Create directory: `toolbox/workflows/my_workflow/` -2. Add `workflow.py` with a Prefect flow -3. Add mandatory `metadata.yaml` -4. Restart backend: `docker-compose restart fuzzforge-backend` +2. Add `workflow.py` with a Temporal workflow (using `@workflow.defn`) +3. Add mandatory `metadata.yaml` with `vertical` field +4. Restart the appropriate worker: `docker-compose -f docker-compose.temporal.yaml restart worker-rust` +5. Worker will automatically discover and register the new workflow ### Adding a New Module 1. Create module in `toolbox/modules/{category}/` 2. Implement `BaseModule` interface -3. Use in workflows via import \ No newline at end of file +3. Use in workflows via import + +### Adding a New Vertical Worker + +1. Create worker directory: `workers/{vertical}/` +2. Create `Dockerfile` with required tools +3. Add worker to `docker-compose.temporal.yaml` +4. Worker will automatically discover workflows with matching `vertical` in metadata \ No newline at end of file diff --git a/backend/benchmarks/README.md b/backend/benchmarks/README.md new file mode 100644 index 0000000..fc29286 --- /dev/null +++ b/backend/benchmarks/README.md @@ -0,0 +1,184 @@ +# FuzzForge Benchmark Suite + +Performance benchmarking infrastructure organized by module category. + +## Directory Structure + +``` +benchmarks/ +ā”œā”€ā”€ conftest.py # Benchmark fixtures +ā”œā”€ā”€ category_configs.py # Category-specific thresholds +ā”œā”€ā”€ by_category/ # Benchmarks organized by category +ā”‚ ā”œā”€ā”€ fuzzer/ +ā”‚ ā”‚ ā”œā”€ā”€ bench_cargo_fuzz.py +ā”‚ ā”‚ ā””ā”€ā”€ bench_atheris.py +ā”‚ ā”œā”€ā”€ scanner/ +ā”‚ ā”‚ ā””ā”€ā”€ bench_file_scanner.py +ā”‚ ā”œā”€ā”€ secret_detection/ +ā”‚ ā”‚ ā”œā”€ā”€ bench_gitleaks.py +ā”‚ ā”‚ ā””ā”€ā”€ bench_trufflehog.py +ā”‚ ā””ā”€ā”€ analyzer/ +ā”‚ ā””ā”€ā”€ bench_security_analyzer.py +ā”œā”€ā”€ fixtures/ # Benchmark test data +ā”‚ ā”œā”€ā”€ small/ # ~1K LOC +ā”‚ ā”œā”€ā”€ medium/ # ~10K LOC +ā”‚ ā””ā”€ā”€ large/ # ~100K LOC +ā””ā”€ā”€ results/ # Benchmark results (JSON) +``` + +## Module Categories + +### Fuzzer +**Expected Metrics**: execs/sec, coverage_rate, time_to_crash, memory_usage + +**Performance Thresholds**: +- Min 1000 execs/sec +- Max 10s for small projects +- Max 2GB memory + +### Scanner +**Expected Metrics**: files/sec, LOC/sec, findings_count + +**Performance Thresholds**: +- Min 100 files/sec +- Min 10K LOC/sec +- Max 512MB memory + +### Secret Detection +**Expected Metrics**: patterns/sec, precision, recall, F1 + +**Performance Thresholds**: +- Min 90% precision +- Min 95% recall +- Max 5 false positives per 100 secrets + +### Analyzer +**Expected Metrics**: analysis_depth, files/sec, accuracy + +**Performance Thresholds**: +- Min 10 files/sec (deep analysis) +- Min 85% accuracy +- Max 2GB memory + +## Running Benchmarks + +### All Benchmarks +```bash +cd backend +pytest benchmarks/ --benchmark-only -v +``` + +### Specific Category +```bash +pytest benchmarks/by_category/fuzzer/ --benchmark-only -v +``` + +### With Comparison +```bash +# Run and save baseline +pytest benchmarks/ --benchmark-only --benchmark-save=baseline + +# Compare against baseline +pytest benchmarks/ --benchmark-only --benchmark-compare=baseline +``` + +### Generate Histogram +```bash +pytest benchmarks/ --benchmark-only --benchmark-histogram=histogram +``` + +## Benchmark Results + +Results are saved as JSON and include: +- Mean execution time +- Standard deviation +- Min/Max values +- Iterations per second +- Memory usage + +Example output: +``` +------------------------ benchmark: fuzzer -------------------------- +Name Mean StdDev Ops/Sec +bench_cargo_fuzz[discovery] 0.0012s 0.0001s 833.33 +bench_cargo_fuzz[execution] 0.1250s 0.0050s 8.00 +bench_cargo_fuzz[memory] 0.0100s 0.0005s 100.00 +--------------------------------------------------------------------- +``` + +## CI/CD Integration + +Benchmarks run: +- **Nightly**: Full benchmark suite, track trends +- **On PR**: When benchmarks/ or modules/ changed +- **Manual**: Via workflow_dispatch + +### Regression Detection + +Benchmarks automatically fail if: +- Performance degrades >10% +- Memory usage exceeds thresholds +- Throughput drops below minimum + +See `.github/workflows/benchmark.yml` for configuration. + +## Adding New Benchmarks + +### 1. Create benchmark file in category directory +```python +# benchmarks/by_category/fuzzer/bench_new_fuzzer.py + +import pytest +from benchmarks.category_configs import ModuleCategory, get_threshold + +@pytest.mark.benchmark(group="fuzzer") +def test_execution_performance(benchmark, new_fuzzer, test_workspace): + """Benchmark execution speed""" + result = benchmark(new_fuzzer.execute, config, test_workspace) + + # Validate against threshold + threshold = get_threshold(ModuleCategory.FUZZER, "max_execution_time_small") + assert result.execution_time < threshold +``` + +### 2. Update category_configs.py if needed +Add new thresholds or metrics for your module. + +### 3. Run locally +```bash +pytest benchmarks/by_category/fuzzer/bench_new_fuzzer.py --benchmark-only -v +``` + +## Best Practices + +1. **Use mocking** for external dependencies (network, disk I/O) +2. **Fixed iterations** for consistent benchmarking +3. **Warm-up runs** for JIT-compiled code +4. **Category-specific metrics** aligned with module purpose +5. **Realistic fixtures** that represent actual use cases +6. **Memory profiling** using tracemalloc +7. **Compare apples to apples** within the same category + +## Interpreting Results + +### Good Performance +- āœ… Execution time below threshold +- āœ… Memory usage within limits +- āœ… Throughput meets minimum +- āœ… <5% variance across runs + +### Performance Issues +- āš ļø Execution time 10-20% over threshold +- āŒ Execution time >20% over threshold +- āŒ Memory leaks (increasing over iterations) +- āŒ High variance (>10%) indicates instability + +## Tracking Performance Over Time + +Benchmark results are stored as artifacts with: +- Commit SHA +- Timestamp +- Environment details (Python version, OS) +- Full metrics + +Use these to track long-term performance trends and detect gradual degradation. diff --git a/backend/benchmarks/by_category/fuzzer/bench_cargo_fuzz.py b/backend/benchmarks/by_category/fuzzer/bench_cargo_fuzz.py new file mode 100644 index 0000000..0cd97ca --- /dev/null +++ b/backend/benchmarks/by_category/fuzzer/bench_cargo_fuzz.py @@ -0,0 +1,221 @@ +""" +Benchmarks for CargoFuzzer module + +Tests performance characteristics of Rust fuzzing: +- Execution throughput (execs/sec) +- Coverage rate +- Memory efficiency +- Time to first crash +""" + +import pytest +import asyncio +from pathlib import Path +from unittest.mock import AsyncMock, patch +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "toolbox")) + +from modules.fuzzer.cargo_fuzzer import CargoFuzzer +from benchmarks.category_configs import ModuleCategory, get_threshold + + +@pytest.fixture +def cargo_fuzzer(): + """Create CargoFuzzer instance for benchmarking""" + return CargoFuzzer() + + +@pytest.fixture +def benchmark_config(): + """Benchmark-optimized configuration""" + return { + "target_name": None, + "max_iterations": 10000, # Fixed iterations for consistent benchmarking + "timeout_seconds": 30, + "sanitizer": "address" + } + + +@pytest.fixture +def mock_rust_workspace(tmp_path): + """Create a minimal Rust workspace for benchmarking""" + workspace = tmp_path / "rust_project" + workspace.mkdir() + + # Cargo.toml + (workspace / "Cargo.toml").write_text("""[package] +name = "bench_project" +version = "0.1.0" +edition = "2021" +""") + + # src/lib.rs + src = workspace / "src" + src.mkdir() + (src / "lib.rs").write_text(""" +pub fn benchmark_function(data: &[u8]) -> Vec { + data.to_vec() +} +""") + + # fuzz structure + fuzz = workspace / "fuzz" + fuzz.mkdir() + (fuzz / "Cargo.toml").write_text("""[package] +name = "bench_project-fuzz" +version = "0.0.0" +edition = "2021" + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.bench_project] +path = ".." + +[[bin]] +name = "fuzz_target_1" +path = "fuzz_targets/fuzz_target_1.rs" +""") + + targets = fuzz / "fuzz_targets" + targets.mkdir() + (targets / "fuzz_target_1.rs").write_text("""#![no_main] +use libfuzzer_sys::fuzz_target; +use bench_project::benchmark_function; + +fuzz_target!(|data: &[u8]| { + let _ = benchmark_function(data); +}); +""") + + return workspace + + +class TestCargoFuzzerPerformance: + """Benchmark CargoFuzzer performance metrics""" + + @pytest.mark.benchmark(group="fuzzer") + def test_target_discovery_performance(self, benchmark, cargo_fuzzer, mock_rust_workspace): + """Benchmark fuzz target discovery speed""" + def discover(): + return asyncio.run(cargo_fuzzer._discover_fuzz_targets(mock_rust_workspace)) + + result = benchmark(discover) + assert len(result) > 0 + + @pytest.mark.benchmark(group="fuzzer") + def test_config_validation_performance(self, benchmark, cargo_fuzzer, benchmark_config): + """Benchmark configuration validation speed""" + result = benchmark(cargo_fuzzer.validate_config, benchmark_config) + assert result is True + + @pytest.mark.benchmark(group="fuzzer") + def test_module_initialization_performance(self, benchmark): + """Benchmark module instantiation time""" + def init_module(): + return CargoFuzzer() + + module = benchmark(init_module) + assert module is not None + + +class TestCargoFuzzerThroughput: + """Benchmark execution throughput""" + + @pytest.mark.benchmark(group="fuzzer") + def test_execution_throughput(self, benchmark, cargo_fuzzer, mock_rust_workspace, benchmark_config): + """Benchmark fuzzing execution throughput""" + + # Mock actual fuzzing to focus on orchestration overhead + async def mock_run(workspace, target, config, callback): + # Simulate 10K execs at 1000 execs/sec + if callback: + await callback({ + "total_execs": 10000, + "execs_per_sec": 1000.0, + "crashes": 0, + "coverage": 50, + "corpus_size": 10, + "elapsed_time": 10 + }) + return [], {"total_executions": 10000, "execution_time": 10.0} + + with patch.object(cargo_fuzzer, '_build_fuzz_target', new_callable=AsyncMock, return_value=True): + with patch.object(cargo_fuzzer, '_run_fuzzing', side_effect=mock_run): + with patch.object(cargo_fuzzer, '_parse_crash_artifacts', new_callable=AsyncMock, return_value=[]): + def run_fuzzer(): + # Run in new event loop + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete( + cargo_fuzzer.execute(benchmark_config, mock_rust_workspace) + ) + finally: + loop.close() + + result = benchmark(run_fuzzer) + assert result.status == "success" + + # Verify performance threshold + threshold = get_threshold(ModuleCategory.FUZZER, "max_execution_time_small") + assert result.execution_time < threshold, \ + f"Execution time {result.execution_time}s exceeds threshold {threshold}s" + + +class TestCargoFuzzerMemory: + """Benchmark memory efficiency""" + + @pytest.mark.benchmark(group="fuzzer") + def test_memory_overhead(self, benchmark, cargo_fuzzer, mock_rust_workspace, benchmark_config): + """Benchmark memory usage during execution""" + import tracemalloc + + def measure_memory(): + tracemalloc.start() + + # Simulate operations + cargo_fuzzer.validate_config(benchmark_config) + asyncio.run(cargo_fuzzer._discover_fuzz_targets(mock_rust_workspace)) + + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + return peak / 1024 / 1024 # Convert to MB + + peak_mb = benchmark(measure_memory) + + # Check against threshold + max_memory = get_threshold(ModuleCategory.FUZZER, "max_memory_mb") + assert peak_mb < max_memory, \ + f"Peak memory {peak_mb:.2f}MB exceeds threshold {max_memory}MB" + + +class TestCargoFuzzerScalability: + """Benchmark scalability characteristics""" + + @pytest.mark.benchmark(group="fuzzer") + def test_multiple_target_discovery(self, benchmark, cargo_fuzzer, tmp_path): + """Benchmark discovery with multiple targets""" + workspace = tmp_path / "multi_target" + workspace.mkdir() + + # Create workspace with 10 fuzz targets + (workspace / "Cargo.toml").write_text("[package]\nname = \"test\"\nversion = \"0.1.0\"\nedition = \"2021\"") + src = workspace / "src" + src.mkdir() + (src / "lib.rs").write_text("pub fn test() {}") + + fuzz = workspace / "fuzz" + fuzz.mkdir() + targets = fuzz / "fuzz_targets" + targets.mkdir() + + for i in range(10): + (targets / f"fuzz_target_{i}.rs").write_text("// Target") + + def discover(): + return asyncio.run(cargo_fuzzer._discover_fuzz_targets(workspace)) + + result = benchmark(discover) + assert len(result) == 10 diff --git a/backend/benchmarks/by_category/secret_detection/README.md b/backend/benchmarks/by_category/secret_detection/README.md new file mode 100644 index 0000000..9fd437d --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/README.md @@ -0,0 +1,240 @@ +# Secret Detection Benchmarks + +Comprehensive benchmarking suite comparing secret detection tools via complete workflow execution: +- **Gitleaks** - Fast pattern-based detection +- **TruffleHog** - Entropy analysis with verification +- **LLM Detector** - AI-powered semantic analysis (gpt-4o-mini, gpt-5-mini) + +## Quick Start + +### Run All Comparisons + +```bash +cd backend +python benchmarks/by_category/secret_detection/compare_tools.py +``` + +This will run all workflows on `test_projects/secret_detection_benchmark/` and generate comparison reports. + +### Run Benchmark Tests + +```bash +# All benchmarks (Gitleaks, TruffleHog, LLM with 3 models) +pytest benchmarks/by_category/secret_detection/bench_comparison.py --benchmark-only -v + +# Specific tool only +pytest benchmarks/by_category/secret_detection/bench_comparison.py::TestSecretDetectionComparison::test_gitleaks_workflow --benchmark-only -v + +# Performance tests only +pytest benchmarks/by_category/secret_detection/bench_comparison.py::TestSecretDetectionPerformance --benchmark-only -v +``` + +## Ground Truth Dataset + +**Controlled Benchmark** (`test_projects/secret_detection_benchmark/`) + +**Exactly 32 documented secrets** for accurate precision/recall testing: +- **12 Easy**: Standard patterns (AWS keys, GitHub PATs, Stripe keys, SSH keys) +- **10 Medium**: Obfuscated (Base64, hex, concatenated, in comments, Unicode) +- **10 Hard**: Well hidden (ROT13, binary, XOR, reversed, template strings, regex patterns) + +All secrets documented in `secret_detection_benchmark_GROUND_TRUTH.json` with exact file paths and line numbers. + +See `test_projects/secret_detection_benchmark/README.md` for details. + +## Metrics Measured + +### Accuracy Metrics +- **Precision**: TP / (TP + FP) - How many detected secrets are real? +- **Recall**: TP / (TP + FN) - How many real secrets were found? +- **F1 Score**: Harmonic mean of precision and recall +- **False Positive Rate**: FP / Total Detected + +### Performance Metrics +- **Execution Time**: Total time to scan all files +- **Throughput**: Files/secrets scanned per second +- **Memory Usage**: Peak memory during execution + +### Thresholds (from `category_configs.py`) +- Minimum Precision: 90% +- Minimum Recall: 95% +- Max Execution Time (small): 2.0s +- Max False Positives: 5 per 100 secrets + +## Tool Comparison + +### Gitleaks +**Strengths:** +- Fastest execution +- Git-aware (commit history scanning) +- Low false positive rate +- No API required +- Works offline + +**Weaknesses:** +- Pattern-based only +- May miss obfuscated secrets +- Limited to known patterns + +### TruffleHog +**Strengths:** +- Secret verification (validates if active) +- High detection rate with entropy analysis +- Multiple detectors (600+ secret types) +- Catches high-entropy strings + +**Weaknesses:** +- Slower than Gitleaks +- Higher false positive rate +- Verification requires network calls + +### LLM Detector +**Strengths:** +- Semantic understanding of context +- Catches novel/custom secret patterns +- Can reason about what "looks like" a secret +- Multiple model options (GPT-4, Claude, etc.) +- Understands code context + +**Weaknesses:** +- Slowest (API latency + LLM processing) +- Most expensive (LLM API costs) +- Requires A2A agent infrastructure +- Accuracy varies by model +- May miss well-disguised secrets + +## Results Directory + +After running comparisons, results are saved to: +``` +benchmarks/by_category/secret_detection/results/ +ā”œā”€ā”€ comparison_report.md # Human-readable comparison with: +ā”‚ # - Summary table with secrets/files/avg per file/time +ā”‚ # - Agreement analysis (secrets found by N tools) +ā”‚ # - Tool agreement matrix (overlap between pairs) +ā”‚ # - Per-file detailed comparison table +ā”‚ # - File type breakdown +ā”‚ # - Files analyzed by each tool +ā”‚ # - Overlap analysis and performance summary +ā””ā”€ā”€ comparison_results.json # Machine-readable data with findings_by_file +``` + +## Latest Benchmark Results + +Run the benchmark to generate results: +```bash +cd backend +python benchmarks/by_category/secret_detection/compare_tools.py +``` + +Results are saved to `results/comparison_report.md` with: +- Summary table (secrets found, files scanned, time) +- Agreement analysis (how many tools found each secret) +- Tool agreement matrix (overlap between tools) +- Per-file detailed comparison +- File type breakdown + +## CI/CD Integration + +Add to your CI pipeline: + +```yaml +# .github/workflows/benchmark-secrets.yml +name: Secret Detection Benchmark + +on: + schedule: + - cron: '0 0 * * 0' # Weekly + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r backend/requirements.txt + pip install pytest-benchmark + + - name: Run benchmarks + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + cd backend + pytest benchmarks/by_category/secret_detection/bench_comparison.py \ + --benchmark-only \ + --benchmark-json=results.json \ + --gitguardian-api-key + + - name: Upload results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: backend/results.json +``` + +## Adding New Tools + +To benchmark a new secret detection tool: + +1. Create module in `toolbox/modules/secret_detection/` +2. Register in `__init__.py` +3. Add to `compare_tools.py` in `run_all_tools()` +4. Add test in `bench_comparison.py` + +## Interpreting Results + +### High Precision, Low Recall +Tool is conservative - few false positives but misses secrets. +**Use case**: Production environments where false positives are costly. + +### Low Precision, High Recall +Tool is aggressive - finds most secrets but many false positives. +**Use case**: Initial scans where manual review is acceptable. + +### Balanced (High F1) +Tool has good balance of precision and recall. +**Use case**: General purpose scanning. + +### Fast Execution +Suitable for CI/CD pipelines and pre-commit hooks. + +### Slow but Accurate +Better for comprehensive security audits. + +## Best Practices + +1. **Use multiple tools**: Each has strengths/weaknesses +2. **Combine results**: Union of all findings for maximum coverage +3. **Filter intelligently**: Remove known false positives +4. **Verify findings**: Check if secrets are actually valid +5. **Track over time**: Monitor precision/recall trends +6. **Update regularly**: Patterns evolve, tools improve + +## Troubleshooting + +### GitGuardian Tests Skipped +- Set `GITGUARDIAN_API_KEY` environment variable +- Use `--gitguardian-api-key` flag + +### LLM Tests Skipped +- Ensure A2A agent is running +- Check agent URL in config +- Use `--llm-enabled` flag + +### Low Recall +- Check if ground truth is up to date +- Verify tool is configured correctly +- Review missed secrets manually + +### High False Positives +- Adjust tool sensitivity +- Add exclusion patterns +- Review false positive list diff --git a/backend/benchmarks/by_category/secret_detection/bench_comparison.py b/backend/benchmarks/by_category/secret_detection/bench_comparison.py new file mode 100644 index 0000000..1dc040e --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/bench_comparison.py @@ -0,0 +1,285 @@ +""" +Secret Detection Tool Comparison Benchmark + +Compares Gitleaks, TruffleHog, and LLM-based detection +on the vulnerable_app ground truth dataset via workflow execution. +""" + +import pytest +import json +from pathlib import Path +from typing import Dict, List, Any +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "sdk" / "src")) + +from fuzzforge_sdk import FuzzForgeClient +from benchmarks.category_configs import ModuleCategory, get_threshold + + +@pytest.fixture +def target_path(): + """Path to vulnerable_app""" + path = Path(__file__).parent.parent.parent.parent.parent / "test_projects" / "vulnerable_app" + assert path.exists(), f"Target not found: {path}" + return path + + +@pytest.fixture +def ground_truth(target_path): + """Load ground truth data""" + metadata_file = target_path / "SECRETS_GROUND_TRUTH.json" + assert metadata_file.exists(), f"Ground truth not found: {metadata_file}" + + with open(metadata_file) as f: + return json.load(f) + + +@pytest.fixture +def sdk_client(): + """FuzzForge SDK client""" + client = FuzzForgeClient(base_url="http://localhost:8000") + yield client + client.close() + + +def calculate_metrics(sarif_results: List[Dict], ground_truth: Dict[str, Any]) -> Dict[str, float]: + """Calculate precision, recall, and F1 score""" + + # Extract expected secrets from ground truth + expected_secrets = set() + for file_info in ground_truth["files"]: + if "secrets" in file_info: + for secret in file_info["secrets"]: + expected_secrets.add((file_info["filename"], secret["line"])) + + # Extract detected secrets from SARIF + detected_secrets = set() + for result in sarif_results: + locations = result.get("locations", []) + for location in locations: + physical_location = location.get("physicalLocation", {}) + artifact_location = physical_location.get("artifactLocation", {}) + region = physical_location.get("region", {}) + + uri = artifact_location.get("uri", "") + line = region.get("startLine", 0) + + if uri and line: + file_path = Path(uri) + filename = file_path.name + detected_secrets.add((filename, line)) + # Also try with relative path + if len(file_path.parts) > 1: + rel_path = str(Path(*file_path.parts[-2:])) + detected_secrets.add((rel_path, line)) + + # Calculate metrics + true_positives = len(expected_secrets & detected_secrets) + false_positives = len(detected_secrets - expected_secrets) + false_negatives = len(expected_secrets - detected_secrets) + + precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0 + recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0 + f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 + + return { + "precision": precision, + "recall": recall, + "f1": f1, + "true_positives": true_positives, + "false_positives": false_positives, + "false_negatives": false_negatives + } + + +class TestSecretDetectionComparison: + """Compare all secret detection tools""" + + @pytest.mark.benchmark(group="secret_detection") + def test_gitleaks_workflow(self, benchmark, sdk_client, target_path, ground_truth): + """Benchmark Gitleaks workflow accuracy and performance""" + + def run_gitleaks(): + run = sdk_client.submit_workflow_with_upload( + workflow_name="gitleaks_detection", + target_path=str(target_path), + parameters={ + "scan_mode": "detect", + "no_git": True, + "redact": False + } + ) + + result = sdk_client.wait_for_completion(run.run_id, timeout=300) + assert result.status == "completed", f"Workflow failed: {result.status}" + + findings = sdk_client.get_run_findings(run.run_id) + assert findings and findings.sarif, "No findings returned" + + return findings + + findings = benchmark(run_gitleaks) + + # Extract SARIF results + sarif_results = [] + for run_data in findings.sarif.get("runs", []): + sarif_results.extend(run_data.get("results", [])) + + # Calculate metrics + metrics = calculate_metrics(sarif_results, ground_truth) + + # Log results + print(f"\n=== Gitleaks Workflow Results ===") + print(f"Precision: {metrics['precision']:.2%}") + print(f"Recall: {metrics['recall']:.2%}") + print(f"F1 Score: {metrics['f1']:.2%}") + print(f"True Positives: {metrics['true_positives']}") + print(f"False Positives: {metrics['false_positives']}") + print(f"False Negatives: {metrics['false_negatives']}") + print(f"Findings Count: {len(sarif_results)}") + + # Assert meets thresholds + min_precision = get_threshold(ModuleCategory.SECRET_DETECTION, "min_precision") + min_recall = get_threshold(ModuleCategory.SECRET_DETECTION, "min_recall") + + assert metrics['precision'] >= min_precision, \ + f"Precision {metrics['precision']:.2%} below threshold {min_precision:.2%}" + assert metrics['recall'] >= min_recall, \ + f"Recall {metrics['recall']:.2%} below threshold {min_recall:.2%}" + + @pytest.mark.benchmark(group="secret_detection") + def test_trufflehog_workflow(self, benchmark, sdk_client, target_path, ground_truth): + """Benchmark TruffleHog workflow accuracy and performance""" + + def run_trufflehog(): + run = sdk_client.submit_workflow_with_upload( + workflow_name="trufflehog_detection", + target_path=str(target_path), + parameters={ + "verify": False, + "max_depth": 10 + } + ) + + result = sdk_client.wait_for_completion(run.run_id, timeout=300) + assert result.status == "completed", f"Workflow failed: {result.status}" + + findings = sdk_client.get_run_findings(run.run_id) + assert findings and findings.sarif, "No findings returned" + + return findings + + findings = benchmark(run_trufflehog) + + sarif_results = [] + for run_data in findings.sarif.get("runs", []): + sarif_results.extend(run_data.get("results", [])) + + metrics = calculate_metrics(sarif_results, ground_truth) + + print(f"\n=== TruffleHog Workflow Results ===") + print(f"Precision: {metrics['precision']:.2%}") + print(f"Recall: {metrics['recall']:.2%}") + print(f"F1 Score: {metrics['f1']:.2%}") + print(f"True Positives: {metrics['true_positives']}") + print(f"False Positives: {metrics['false_positives']}") + print(f"False Negatives: {metrics['false_negatives']}") + print(f"Findings Count: {len(sarif_results)}") + + min_precision = get_threshold(ModuleCategory.SECRET_DETECTION, "min_precision") + min_recall = get_threshold(ModuleCategory.SECRET_DETECTION, "min_recall") + + assert metrics['precision'] >= min_precision + assert metrics['recall'] >= min_recall + + @pytest.mark.benchmark(group="secret_detection") + @pytest.mark.parametrize("model", [ + "gpt-4o-mini", + "gpt-4o", + "claude-3-5-sonnet-20241022" + ]) + def test_llm_workflow(self, benchmark, sdk_client, target_path, ground_truth, model): + """Benchmark LLM workflow with different models""" + + def run_llm(): + provider = "openai" if "gpt" in model else "anthropic" + + run = sdk_client.submit_workflow_with_upload( + workflow_name="llm_secret_detection", + target_path=str(target_path), + parameters={ + "agent_url": "http://fuzzforge-task-agent:8000/a2a/litellm_agent", + "llm_model": model, + "llm_provider": provider, + "max_files": 20, + "timeout": 60 + } + ) + + result = sdk_client.wait_for_completion(run.run_id, timeout=300) + assert result.status == "completed", f"Workflow failed: {result.status}" + + findings = sdk_client.get_run_findings(run.run_id) + assert findings and findings.sarif, "No findings returned" + + return findings + + findings = benchmark(run_llm) + + sarif_results = [] + for run_data in findings.sarif.get("runs", []): + sarif_results.extend(run_data.get("results", [])) + + metrics = calculate_metrics(sarif_results, ground_truth) + + print(f"\n=== LLM ({model}) Workflow Results ===") + print(f"Precision: {metrics['precision']:.2%}") + print(f"Recall: {metrics['recall']:.2%}") + print(f"F1 Score: {metrics['f1']:.2%}") + print(f"True Positives: {metrics['true_positives']}") + print(f"False Positives: {metrics['false_positives']}") + print(f"False Negatives: {metrics['false_negatives']}") + print(f"Findings Count: {len(sarif_results)}") + + +class TestSecretDetectionPerformance: + """Performance benchmarks for each tool""" + + @pytest.mark.benchmark(group="secret_detection") + def test_gitleaks_performance(self, benchmark, sdk_client, target_path): + """Benchmark Gitleaks workflow execution speed""" + + def run(): + run = sdk_client.submit_workflow_with_upload( + workflow_name="gitleaks_detection", + target_path=str(target_path), + parameters={"scan_mode": "detect", "no_git": True} + ) + result = sdk_client.wait_for_completion(run.run_id, timeout=300) + return result + + result = benchmark(run) + + max_time = get_threshold(ModuleCategory.SECRET_DETECTION, "max_execution_time_small") + # Note: Workflow execution time includes orchestration overhead + # so we allow 2x the module threshold + assert result.execution_time < max_time * 2 + + @pytest.mark.benchmark(group="secret_detection") + def test_trufflehog_performance(self, benchmark, sdk_client, target_path): + """Benchmark TruffleHog workflow execution speed""" + + def run(): + run = sdk_client.submit_workflow_with_upload( + workflow_name="trufflehog_detection", + target_path=str(target_path), + parameters={"verify": False} + ) + result = sdk_client.wait_for_completion(run.run_id, timeout=300) + return result + + result = benchmark(run) + + max_time = get_threshold(ModuleCategory.SECRET_DETECTION, "max_execution_time_small") + assert result.execution_time < max_time * 2 diff --git a/backend/benchmarks/by_category/secret_detection/compare_tools.py b/backend/benchmarks/by_category/secret_detection/compare_tools.py new file mode 100644 index 0000000..ae03c99 --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/compare_tools.py @@ -0,0 +1,547 @@ +""" +Secret Detection Tools Comparison Report Generator + +Generates comparison reports showing strengths/weaknesses of each tool. +Uses workflow execution via SDK to test complete pipeline. +""" + +import asyncio +import json +import time +from pathlib import Path +from typing import Dict, List, Any, Optional +from dataclasses import dataclass, asdict +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "sdk" / "src")) + +from fuzzforge_sdk import FuzzForgeClient + + +@dataclass +class ToolResult: + """Results from running a tool""" + tool_name: str + execution_time: float + findings_count: int + findings_by_file: Dict[str, List[int]] # file_path -> [line_numbers] + unique_files: int + unique_locations: int # unique (file, line) pairs + secret_density: float # average secrets per file + file_types: Dict[str, int] # file extension -> count of files with secrets + + +class SecretDetectionComparison: + """Compare secret detection tools""" + + def __init__(self, target_path: Path, api_url: str = "http://localhost:8000"): + self.target_path = target_path + self.client = FuzzForgeClient(base_url=api_url) + + async def run_workflow(self, workflow_name: str, tool_name: str, config: Dict[str, Any] = None) -> Optional[ToolResult]: + """Run a workflow and extract findings""" + print(f"\nRunning {tool_name} workflow...") + + start_time = time.time() + + try: + # Start workflow + run = self.client.submit_workflow_with_upload( + workflow_name=workflow_name, + target_path=str(self.target_path), + parameters=config or {} + ) + + print(f" Started run: {run.run_id}") + + # Wait for completion (up to 30 minutes for slow LLMs) + print(f" Waiting for completion...") + result = self.client.wait_for_completion(run.run_id, timeout=1800) + + execution_time = time.time() - start_time + + if result.status != "COMPLETED": + print(f"āŒ {tool_name} workflow failed: {result.status}") + return None + + # Get findings from SARIF + findings = self.client.get_run_findings(run.run_id) + + if not findings or not findings.sarif: + print(f"āš ļø {tool_name} produced no findings") + return None + + # Extract results from SARIF and group by file + findings_by_file = {} + unique_locations = set() + + for run_data in findings.sarif.get("runs", []): + for result in run_data.get("results", []): + locations = result.get("locations", []) + for location in locations: + physical_location = location.get("physicalLocation", {}) + artifact_location = physical_location.get("artifactLocation", {}) + region = physical_location.get("region", {}) + + uri = artifact_location.get("uri", "") + line = region.get("startLine", 0) + + if uri and line: + if uri not in findings_by_file: + findings_by_file[uri] = [] + findings_by_file[uri].append(line) + unique_locations.add((uri, line)) + + # Sort line numbers for each file + for file_path in findings_by_file: + findings_by_file[file_path] = sorted(set(findings_by_file[file_path])) + + # Calculate file type distribution + file_types = {} + for file_path in findings_by_file: + ext = Path(file_path).suffix or Path(file_path).name # Use full name for files like .env + if ext.startswith('.'): + file_types[ext] = file_types.get(ext, 0) + 1 + else: + file_types['[no extension]'] = file_types.get('[no extension]', 0) + 1 + + # Calculate secret density + secret_density = len(unique_locations) / len(findings_by_file) if findings_by_file else 0 + + print(f" āœ“ Found {len(unique_locations)} secrets in {len(findings_by_file)} files (avg {secret_density:.1f} per file)") + + return ToolResult( + tool_name=tool_name, + execution_time=execution_time, + findings_count=len(unique_locations), + findings_by_file=findings_by_file, + unique_files=len(findings_by_file), + unique_locations=len(unique_locations), + secret_density=secret_density, + file_types=file_types + ) + + except Exception as e: + print(f"āŒ {tool_name} error: {e}") + return None + + + async def run_all_tools(self, llm_models: List[str] = None) -> List[ToolResult]: + """Run all available tools""" + results = [] + + if llm_models is None: + llm_models = ["gpt-4o-mini"] + + # Gitleaks + result = await self.run_workflow("gitleaks_detection", "Gitleaks", { + "scan_mode": "detect", + "no_git": True, + "redact": False + }) + if result: + results.append(result) + + # TruffleHog + result = await self.run_workflow("trufflehog_detection", "TruffleHog", { + "verify": False, + "max_depth": 10 + }) + if result: + results.append(result) + + # LLM Detector with multiple models + for model in llm_models: + tool_name = f"LLM ({model})" + result = await self.run_workflow("llm_secret_detection", tool_name, { + "agent_url": "http://fuzzforge-task-agent:8000/a2a/litellm_agent", + "llm_model": model, + "llm_provider": "openai" if "gpt" in model else "anthropic", + "max_files": 20, + "timeout": 60, + "file_patterns": [ + "*.py", "*.js", "*.ts", "*.java", "*.go", "*.env", "*.yaml", "*.yml", + "*.json", "*.xml", "*.ini", "*.sql", "*.properties", "*.sh", "*.bat", + "*.config", "*.conf", "*.toml", "*id_rsa*", "*.txt" + ] + }) + if result: + results.append(result) + + return results + + def _calculate_agreement_matrix(self, results: List[ToolResult]) -> Dict[str, Dict[str, int]]: + """Calculate overlap matrix showing common secrets between tool pairs""" + matrix = {} + + for i, result1 in enumerate(results): + matrix[result1.tool_name] = {} + # Convert to set of (file, line) tuples + secrets1 = set() + for file_path, lines in result1.findings_by_file.items(): + for line in lines: + secrets1.add((file_path, line)) + + for result2 in results: + secrets2 = set() + for file_path, lines in result2.findings_by_file.items(): + for line in lines: + secrets2.add((file_path, line)) + + # Count common secrets + common = len(secrets1 & secrets2) + matrix[result1.tool_name][result2.tool_name] = common + + return matrix + + def _get_per_file_comparison(self, results: List[ToolResult]) -> Dict[str, Dict[str, int]]: + """Get per-file breakdown of findings across all tools""" + all_files = set() + for result in results: + all_files.update(result.findings_by_file.keys()) + + comparison = {} + for file_path in sorted(all_files): + comparison[file_path] = {} + for result in results: + comparison[file_path][result.tool_name] = len(result.findings_by_file.get(file_path, [])) + + return comparison + + def _get_agreement_stats(self, results: List[ToolResult]) -> Dict[int, int]: + """Calculate how many secrets are found by 1, 2, 3, or all tools""" + # Collect all unique (file, line) pairs across all tools + all_secrets = {} # (file, line) -> list of tools that found it + + for result in results: + for file_path, lines in result.findings_by_file.items(): + for line in lines: + key = (file_path, line) + if key not in all_secrets: + all_secrets[key] = [] + all_secrets[key].append(result.tool_name) + + # Count by number of tools + agreement_counts = {} + for secret, tools in all_secrets.items(): + count = len(set(tools)) # Unique tools + agreement_counts[count] = agreement_counts.get(count, 0) + 1 + + return agreement_counts + + def generate_markdown_report(self, results: List[ToolResult]) -> str: + """Generate markdown comparison report""" + report = [] + report.append("# Secret Detection Tools Comparison\n") + report.append(f"**Target**: {self.target_path.name}") + report.append(f"**Tools**: {', '.join([r.tool_name for r in results])}\n") + + # Summary table with extended metrics + report.append("\n## Summary\n") + report.append("| Tool | Secrets | Files | Avg/File | Time (s) |") + report.append("|------|---------|-------|----------|----------|") + + for result in results: + report.append( + f"| {result.tool_name} | " + f"{result.findings_count} | " + f"{result.unique_files} | " + f"{result.secret_density:.1f} | " + f"{result.execution_time:.2f} |" + ) + + # Agreement Analysis + agreement_stats = self._get_agreement_stats(results) + report.append("\n## Agreement Analysis\n") + report.append("Secrets found by different numbers of tools:\n") + for num_tools in sorted(agreement_stats.keys(), reverse=True): + count = agreement_stats[num_tools] + if num_tools == len(results): + report.append(f"- **All {num_tools} tools agree**: {count} secrets") + elif num_tools == 1: + report.append(f"- **Only 1 tool found**: {count} secrets") + else: + report.append(f"- **{num_tools} tools agree**: {count} secrets") + + # Agreement Matrix + agreement_matrix = self._calculate_agreement_matrix(results) + report.append("\n## Tool Agreement Matrix\n") + report.append("Number of common secrets found by tool pairs:\n") + + # Header row + header = "| Tool |" + separator = "|------|" + for result in results: + short_name = result.tool_name.replace("LLM (", "").replace(")", "") + header += f" {short_name} |" + separator += "------|" + report.append(header) + report.append(separator) + + # Data rows + for result in results: + short_name = result.tool_name.replace("LLM (", "").replace(")", "") + row = f"| {short_name} |" + for result2 in results: + count = agreement_matrix[result.tool_name][result2.tool_name] + row += f" {count} |" + report.append(row) + + # Per-File Comparison + per_file = self._get_per_file_comparison(results) + report.append("\n## Per-File Detailed Comparison\n") + report.append("Secrets found per file by each tool:\n") + + # Header + header = "| File |" + separator = "|------|" + for result in results: + short_name = result.tool_name.replace("LLM (", "").replace(")", "") + header += f" {short_name} |" + separator += "------|" + header += " Total |" + separator += "------|" + report.append(header) + report.append(separator) + + # Show top 15 files by total findings + file_totals = [(f, sum(counts.values())) for f, counts in per_file.items()] + file_totals.sort(key=lambda x: x[1], reverse=True) + + for file_path, total in file_totals[:15]: + row = f"| `{file_path}` |" + for result in results: + count = per_file[file_path].get(result.tool_name, 0) + row += f" {count} |" + row += f" **{total}** |" + report.append(row) + + if len(file_totals) > 15: + report.append(f"| ... and {len(file_totals) - 15} more files | ... | ... | ... | ... | ... |") + + # File Type Breakdown + report.append("\n## File Type Breakdown\n") + all_extensions = set() + for result in results: + all_extensions.update(result.file_types.keys()) + + if all_extensions: + header = "| Type |" + separator = "|------|" + for result in results: + short_name = result.tool_name.replace("LLM (", "").replace(")", "") + header += f" {short_name} |" + separator += "------|" + report.append(header) + report.append(separator) + + for ext in sorted(all_extensions): + row = f"| `{ext}` |" + for result in results: + count = result.file_types.get(ext, 0) + row += f" {count} files |" + report.append(row) + + # File analysis + report.append("\n## Files Analyzed\n") + + # Collect all unique files across all tools + all_files = set() + for result in results: + all_files.update(result.findings_by_file.keys()) + + report.append(f"**Total unique files with secrets**: {len(all_files)}\n") + + for result in results: + report.append(f"\n### {result.tool_name}\n") + report.append(f"Found secrets in **{result.unique_files} files**:\n") + + # Sort files by number of findings (descending) + sorted_files = sorted( + result.findings_by_file.items(), + key=lambda x: len(x[1]), + reverse=True + ) + + # Show top 10 files + for file_path, lines in sorted_files[:10]: + report.append(f"- `{file_path}`: {len(lines)} secrets (lines: {', '.join(map(str, lines[:5]))}{'...' if len(lines) > 5 else ''})") + + if len(sorted_files) > 10: + report.append(f"- ... and {len(sorted_files) - 10} more files") + + # Overlap analysis + if len(results) >= 2: + report.append("\n## Overlap Analysis\n") + + # Find common files + file_sets = [set(r.findings_by_file.keys()) for r in results] + common_files = set.intersection(*file_sets) if file_sets else set() + + if common_files: + report.append(f"\n**Files found by all tools** ({len(common_files)}):\n") + for file_path in sorted(common_files)[:10]: + report.append(f"- `{file_path}`") + else: + report.append("\n**No files were found by all tools**\n") + + # Find tool-specific files + for i, result in enumerate(results): + unique_to_tool = set(result.findings_by_file.keys()) + for j, other_result in enumerate(results): + if i != j: + unique_to_tool -= set(other_result.findings_by_file.keys()) + + if unique_to_tool: + report.append(f"\n**Unique to {result.tool_name}** ({len(unique_to_tool)} files):\n") + for file_path in sorted(unique_to_tool)[:5]: + report.append(f"- `{file_path}`") + if len(unique_to_tool) > 5: + report.append(f"- ... and {len(unique_to_tool) - 5} more") + + # Ground Truth Analysis (if available) + ground_truth_path = Path(__file__).parent / "secret_detection_benchmark_GROUND_TRUTH.json" + if ground_truth_path.exists(): + report.append("\n## Ground Truth Analysis\n") + try: + with open(ground_truth_path) as f: + gt_data = json.load(f) + + gt_total = gt_data.get("total_secrets", 30) + report.append(f"**Expected secrets**: {gt_total} (documented in ground truth)\n") + + # Build ground truth set of (file, line) tuples + gt_secrets = set() + for secret in gt_data.get("secrets", []): + gt_secrets.add((secret["file"], secret["line"])) + + report.append("### Tool Performance vs Ground Truth\n") + report.append("| Tool | Found | Expected | Recall | Extra Findings |") + report.append("|------|-------|----------|--------|----------------|") + + for result in results: + # Build tool findings set + tool_secrets = set() + for file_path, lines in result.findings_by_file.items(): + for line in lines: + tool_secrets.add((file_path, line)) + + # Calculate metrics + true_positives = len(gt_secrets & tool_secrets) + recall = (true_positives / gt_total * 100) if gt_total > 0 else 0 + extra = len(tool_secrets - gt_secrets) + + report.append( + f"| {result.tool_name} | " + f"{result.findings_count} | " + f"{gt_total} | " + f"{recall:.1f}% | " + f"{extra} |" + ) + + # Analyze LLM extra findings + llm_results = [r for r in results if "LLM" in r.tool_name] + if llm_results: + report.append("\n### LLM Extra Findings Explanation\n") + report.append("LLMs may find more than 30 secrets because they detect:\n") + report.append("- **Split secret components**: Each part of `DB_PASS_PART1 + PART2 + PART3` counted separately") + report.append("- **Join operations**: Lines like `''.join(AWS_SECRET_CHARS)` flagged as additional exposure") + report.append("- **Decoding functions**: Code that reveals secrets (e.g., `base64.b64decode()`, `codecs.decode()`)") + report.append("- **Comment identifiers**: Lines marking secret locations without plaintext values") + report.append("\nThese are *technically correct* detections of secret exposure points, not false positives.") + report.append("The ground truth documents 30 'primary' secrets, but the codebase has additional derivative exposures.\n") + + except Exception as e: + report.append(f"*Could not load ground truth: {e}*\n") + + # Performance summary + if results: + report.append("\n## Performance Summary\n") + most_findings = max(results, key=lambda r: r.findings_count) + most_files = max(results, key=lambda r: r.unique_files) + fastest = min(results, key=lambda r: r.execution_time) + + report.append(f"- **Most secrets found**: {most_findings.tool_name} ({most_findings.findings_count} secrets)") + report.append(f"- **Most files covered**: {most_files.tool_name} ({most_files.unique_files} files)") + report.append(f"- **Fastest**: {fastest.tool_name} ({fastest.execution_time:.2f}s)") + + return "\n".join(report) + + def save_json_report(self, results: List[ToolResult], output_path: Path): + """Save results as JSON""" + data = { + "target_path": str(self.target_path), + "results": [asdict(r) for r in results] + } + + with open(output_path, 'w') as f: + json.dump(data, f, indent=2) + + print(f"\nāœ… JSON report saved to: {output_path}") + + def cleanup(self): + """Cleanup SDK client""" + self.client.close() + + +async def main(): + """Run comparison and generate reports""" + # Get target path (secret_detection_benchmark) + target_path = Path(__file__).parent.parent.parent.parent.parent / "test_projects" / "secret_detection_benchmark" + + if not target_path.exists(): + print(f"āŒ Target not found at: {target_path}") + return 1 + + print("=" * 80) + print("Secret Detection Tools Comparison") + print("=" * 80) + print(f"Target: {target_path}") + + # LLM models to test + llm_models = [ + "gpt-4o-mini", + "gpt-5-mini" + ] + print(f"LLM models: {', '.join(llm_models)}\n") + + # Run comparison + comparison = SecretDetectionComparison(target_path) + + try: + results = await comparison.run_all_tools(llm_models=llm_models) + + if not results: + print("āŒ No tools ran successfully") + return 1 + + # Generate reports + print("\n" + "=" * 80) + markdown_report = comparison.generate_markdown_report(results) + print(markdown_report) + + # Save reports + output_dir = Path(__file__).parent / "results" + output_dir.mkdir(exist_ok=True) + + markdown_path = output_dir / "comparison_report.md" + with open(markdown_path, 'w') as f: + f.write(markdown_report) + print(f"\nāœ… Markdown report saved to: {markdown_path}") + + json_path = output_dir / "comparison_results.json" + comparison.save_json_report(results, json_path) + + print("\n" + "=" * 80) + print("āœ… Comparison complete!") + print("=" * 80) + + return 0 + + finally: + comparison.cleanup() + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + sys.exit(exit_code) diff --git a/backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json b/backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json new file mode 100644 index 0000000..cd6223c --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json @@ -0,0 +1,344 @@ +{ + "description": "Ground truth dataset for secret detection benchmarking - Exactly 32 secrets", + "version": "1.1.0", + "total_secrets": 32, + "secrets_by_difficulty": { + "easy": 12, + "medium": 10, + "hard": 10 + }, + "secrets": [ + { + "id": 1, + "file": ".env", + "line": 3, + "difficulty": "easy", + "type": "aws_access_key", + "value": "AKIAIOSFODNN7EXAMPLE", + "severity": "critical" + }, + { + "id": 2, + "file": ".env", + "line": 4, + "difficulty": "easy", + "type": "aws_secret_access_key", + "value": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "severity": "critical" + }, + { + "id": 3, + "file": "config/settings.py", + "line": 6, + "difficulty": "easy", + "type": "github_pat", + "value": "ghp_vR8jK2mN4pQ6tX9bC3wY7zA1eF5hI8kL", + "severity": "critical" + }, + { + "id": 4, + "file": "config/settings.py", + "line": 9, + "difficulty": "easy", + "type": "stripe_api_key", + "value": "sk_live_51MabcdefghijklmnopqrstuvwxyzABCDEF123456789", + "severity": "critical" + }, + { + "id": 5, + "file": "config/settings.py", + "line": 17, + "difficulty": "easy", + "type": "database_password", + "value": "ProdDB_P@ssw0rd_2024_Secure!", + "severity": "critical" + }, + { + "id": 6, + "file": "src/app.py", + "line": 6, + "difficulty": "easy", + "type": "jwt_secret", + "value": "my-super-secret-jwt-key-do-not-share-2024", + "severity": "critical" + }, + { + "id": 7, + "file": "config/database.yaml", + "line": 7, + "difficulty": "easy", + "type": "azure_storage_key", + "value": "DefaultEndpointsProtocol=https;AccountName=prodstore;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;EndpointSuffix=core.windows.net", + "severity": "critical" + }, + { + "id": 8, + "file": "scripts/webhook.js", + "line": 4, + "difficulty": "easy", + "type": "slack_webhook", + "value": "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXX", + "severity": "high" + }, + { + "id": 9, + "file": "config/app.properties", + "line": 6, + "difficulty": "easy", + "type": "api_key", + "value": "sk_test_4eC39HqLyjWDarjtT1zdp7dc", + "severity": "high" + }, + { + "id": 10, + "file": "id_rsa", + "line": 1, + "difficulty": "easy", + "type": "ssh_private_key", + "value": "-----BEGIN OPENSSH PRIVATE KEY-----", + "severity": "critical" + }, + { + "id": 11, + "file": "config/oauth.json", + "line": 4, + "difficulty": "easy", + "type": "oauth_client_secret", + "value": "GOCSPX-Ab12Cd34Ef56Gh78Ij90Kl12", + "severity": "critical" + }, + { + "id": 12, + "file": "src/Main.java", + "line": 5, + "difficulty": "easy", + "type": "google_oauth_secret", + "value": "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", + "severity": "critical" + }, + { + "id": 13, + "file": "src/config.py", + "line": 7, + "difficulty": "medium", + "type": "aws_access_key_base64", + "value": "QUtJQUlPU0ZPRE5ON0VYQU1QTEU=", + "decoded": "AKIAIOSFODNN7EXAMPLE", + "severity": "critical" + }, + { + "id": 14, + "file": "src/config.py", + "line": 10, + "difficulty": "medium", + "type": "api_token_hex", + "value": "6170695f746f6b656e5f616263313233787977373839", + "decoded": "api_token_abc123xyz789", + "severity": "high" + }, + { + "id": 15, + "file": "src/config.py", + "line": 16, + "difficulty": "medium", + "type": "database_password_concatenated", + "value": "MySecurePassword2024!", + "note": "Built from DB_PASS_PART1 + DB_PASS_PART2 + DB_PASS_PART3", + "severity": "critical" + }, + { + "id": 16, + "file": "scripts/deploy.sh", + "line": 5, + "difficulty": "medium", + "type": "api_key_export", + "value": "sk_prod_1234567890abcdefghijklmnopqrstuvwxyz", + "severity": "critical" + }, + { + "id": 17, + "file": "scripts/deploy.sh", + "line": 11, + "difficulty": "medium", + "type": "database_password_url_encoded", + "value": "mysql://admin:MyP%40ssw0rd%21@db.example.com:3306/prod", + "decoded": "mysql://admin:MyP@ssw0rd!@db.example.com:3306/prod", + "note": "In comment", + "severity": "critical" + }, + { + "id": 18, + "file": "config/keys.yaml", + "line": 6, + "difficulty": "medium", + "type": "rsa_private_key_multiline", + "value": "-----BEGIN RSA PRIVATE KEY-----", + "note": "Multi-line YAML literal block", + "severity": "critical" + }, + { + "id": 19, + "file": "config/keys.yaml", + "line": 11, + "difficulty": "medium", + "type": "api_token_unicode", + "value": "tĆøkęn_Å›Ä™Ć§rėt_įŗƒĆÆth_Å­Ć±ĆÆĆ§Å‘dė_123456", + "severity": "high" + }, + { + "id": 20, + "file": "src/database.sql", + "line": 6, + "difficulty": "medium", + "type": "database_connection_string", + "value": "postgresql://admin:Pr0dDB_S3cr3t_P@ss@db.prod.example.com:5432/prod_db", + "note": "In SQL comment", + "severity": "critical" + }, + { + "id": 21, + "file": "config/legacy.ini", + "line": 3, + "difficulty": "medium", + "type": "database_password", + "value": "L3g@cy_DB_P@ssw0rd_2023", + "severity": "critical" + }, + { + "id": 22, + "file": "config/legacy.ini", + "line": 7, + "difficulty": "medium", + "type": "api_key_commented", + "value": "backup_key_xyz789abc123def456ghi", + "note": "Commented backup key", + "severity": "high" + }, + { + "id": 23, + "file": "src/obfuscated.py", + "line": 7, + "difficulty": "hard", + "type": "stripe_key_rot13", + "value": "fx_yvir_frperg_xrl_12345", + "decoded": "sk_live_secret_key_12345", + "severity": "critical" + }, + { + "id": 24, + "file": "src/obfuscated.py", + "line": 10, + "difficulty": "hard", + "type": "github_token_binary", + "value": "b'\\x67\\x68\\x70\\x5f\\x4d\\x79\\x47\\x69\\x74\\x48\\x75\\x62\\x54\\x6f\\x6b\\x65\\x6e\\x31\\x32\\x33\\x34\\x35\\x36'", + "decoded": "ghp_MyGitHubToken123456", + "severity": "critical" + }, + { + "id": 25, + "file": "src/obfuscated.py", + "line": 13, + "difficulty": "hard", + "type": "aws_secret_char_array", + "value": "['A','W','S','_','S','E','C','R','E','T','_','K','E','Y','_','X','Y','Z','7','8','9']", + "decoded": "AWS_SECRET_KEY_XYZ789", + "severity": "critical" + }, + { + "id": 26, + "file": "src/obfuscated.py", + "line": 17, + "difficulty": "hard", + "type": "api_token_reversed", + "value": "321cba_desrever_nekot_ipa", + "decoded": "api_token_reversed_abc123", + "severity": "high" + }, + { + "id": 27, + "file": "src/advanced.js", + "line": 4, + "difficulty": "hard", + "type": "secret_template_string", + "value": "sk_prod_template_key_xyz", + "note": "Built from template literals", + "severity": "critical" + }, + { + "id": 28, + "file": "src/advanced.js", + "line": 7, + "difficulty": "hard", + "type": "password_in_regex", + "value": "password_regex_secret_789", + "note": "Inside regex pattern", + "severity": "medium" + }, + { + "id": 29, + "file": "src/advanced.js", + "line": 10, + "difficulty": "hard", + "type": "api_key_xor", + "value": "[65,82,90,75,94,91,92,75,93,67,65,90,67,92,75,91,67,95]", + "decoded": "api_xor_secret_key", + "note": "XOR encrypted with key 42", + "severity": "critical" + }, + { + "id": 30, + "file": "src/advanced.js", + "line": 17, + "difficulty": "hard", + "type": "api_key_escaped_json", + "value": "sk_escaped_json_key_456", + "note": "Escaped JSON within string", + "severity": "high" + }, + { + "id": 31, + "file": "src/Crypto.go", + "line": 10, + "difficulty": "hard", + "type": "secret_in_heredoc", + "value": "golang_heredoc_secret_999", + "note": "In heredoc/multi-line string", + "severity": "high" + }, + { + "id": 32, + "file": "src/Crypto.go", + "line": 15, + "difficulty": "hard", + "type": "stripe_key_typo", + "value": "strippe_sk_live_corrected_key", + "decoded": "stripe_sk_live_corrected_key", + "note": "Intentional typo corrected programmatically", + "severity": "critical" + } + ], + "file_summary": { + ".env": 2, + "config/settings.py": 3, + "src/app.py": 1, + "config/database.yaml": 1, + "scripts/webhook.js": 1, + "config/app.properties": 1, + "id_rsa": 1, + "config/oauth.json": 1, + "src/Main.java": 1, + "src/config.py": 3, + "scripts/deploy.sh": 2, + "config/keys.yaml": 2, + "src/database.sql": 1, + "config/legacy.ini": 2, + "src/obfuscated.py": 4, + "src/advanced.js": 4, + "src/Crypto.go": 2 + }, + "notes": { + "easy_secrets": "Standard patterns that any decent secret scanner should detect", + "medium_secrets": "Slightly obfuscated - base64, hex, concatenated, or in comments", + "hard_secrets": "Well hidden - ROT13, binary, XOR, reversed, split across constructs" + } +} diff --git a/backend/benchmarks/category_configs.py b/backend/benchmarks/category_configs.py new file mode 100644 index 0000000..429a68f --- /dev/null +++ b/backend/benchmarks/category_configs.py @@ -0,0 +1,151 @@ +""" +Category-specific benchmark configurations + +Defines expected metrics and performance thresholds for each module category. +""" + +from dataclasses import dataclass +from typing import List, Dict +from enum import Enum + + +class ModuleCategory(str, Enum): + """Module categories for benchmarking""" + FUZZER = "fuzzer" + SCANNER = "scanner" + ANALYZER = "analyzer" + SECRET_DETECTION = "secret_detection" + REPORTER = "reporter" + + +@dataclass +class CategoryBenchmarkConfig: + """Benchmark configuration for a module category""" + category: ModuleCategory + expected_metrics: List[str] + performance_thresholds: Dict[str, float] + description: str + + +# Fuzzer category configuration +FUZZER_CONFIG = CategoryBenchmarkConfig( + category=ModuleCategory.FUZZER, + expected_metrics=[ + "execs_per_sec", + "coverage_rate", + "time_to_first_crash", + "corpus_efficiency", + "execution_time", + "peak_memory_mb" + ], + performance_thresholds={ + "min_execs_per_sec": 1000, # Minimum executions per second + "max_execution_time_small": 10.0, # Max time for small project (seconds) + "max_execution_time_medium": 60.0, # Max time for medium project + "max_memory_mb": 2048, # Maximum memory usage + "min_coverage_rate": 1.0, # Minimum new coverage per second + }, + description="Fuzzing modules: coverage-guided fuzz testing" +) + +# Scanner category configuration +SCANNER_CONFIG = CategoryBenchmarkConfig( + category=ModuleCategory.SCANNER, + expected_metrics=[ + "files_per_sec", + "loc_per_sec", + "execution_time", + "peak_memory_mb", + "findings_count" + ], + performance_thresholds={ + "min_files_per_sec": 100, # Minimum files scanned per second + "min_loc_per_sec": 10000, # Minimum lines of code per second + "max_execution_time_small": 1.0, + "max_execution_time_medium": 10.0, + "max_memory_mb": 512, + }, + description="File scanning modules: fast pattern-based scanning" +) + +# Secret detection category configuration +SECRET_DETECTION_CONFIG = CategoryBenchmarkConfig( + category=ModuleCategory.SECRET_DETECTION, + expected_metrics=[ + "patterns_per_sec", + "precision", + "recall", + "f1_score", + "false_positive_rate", + "execution_time", + "peak_memory_mb" + ], + performance_thresholds={ + "min_patterns_per_sec": 1000, + "min_precision": 0.90, # 90% precision target + "min_recall": 0.95, # 95% recall target + "max_false_positives": 5, # Max false positives per 100 secrets + "max_execution_time_small": 2.0, + "max_execution_time_medium": 20.0, + "max_memory_mb": 1024, + }, + description="Secret detection modules: high precision pattern matching" +) + +# Analyzer category configuration +ANALYZER_CONFIG = CategoryBenchmarkConfig( + category=ModuleCategory.ANALYZER, + expected_metrics=[ + "analysis_depth", + "files_analyzed_per_sec", + "execution_time", + "peak_memory_mb", + "findings_count", + "accuracy" + ], + performance_thresholds={ + "min_files_per_sec": 10, # Slower than scanners due to deep analysis + "max_execution_time_small": 5.0, + "max_execution_time_medium": 60.0, + "max_memory_mb": 2048, + "min_accuracy": 0.85, # 85% accuracy target + }, + description="Code analysis modules: deep semantic analysis" +) + +# Reporter category configuration +REPORTER_CONFIG = CategoryBenchmarkConfig( + category=ModuleCategory.REPORTER, + expected_metrics=[ + "report_generation_time", + "findings_per_sec", + "peak_memory_mb" + ], + performance_thresholds={ + "max_report_time_100_findings": 1.0, # Max 1 second for 100 findings + "max_report_time_1000_findings": 10.0, # Max 10 seconds for 1000 findings + "max_memory_mb": 256, + }, + description="Reporting modules: fast report generation" +) + + +# Category configurations map +CATEGORY_CONFIGS = { + ModuleCategory.FUZZER: FUZZER_CONFIG, + ModuleCategory.SCANNER: SCANNER_CONFIG, + ModuleCategory.SECRET_DETECTION: SECRET_DETECTION_CONFIG, + ModuleCategory.ANALYZER: ANALYZER_CONFIG, + ModuleCategory.REPORTER: REPORTER_CONFIG, +} + + +def get_category_config(category: ModuleCategory) -> CategoryBenchmarkConfig: + """Get benchmark configuration for a category""" + return CATEGORY_CONFIGS[category] + + +def get_threshold(category: ModuleCategory, metric: str) -> float: + """Get performance threshold for a specific metric""" + config = get_category_config(category) + return config.performance_thresholds.get(metric, 0.0) diff --git a/backend/benchmarks/conftest.py b/backend/benchmarks/conftest.py new file mode 100644 index 0000000..2710fb4 --- /dev/null +++ b/backend/benchmarks/conftest.py @@ -0,0 +1,60 @@ +""" +Benchmark fixtures and configuration +""" + +import sys +from pathlib import Path +import pytest + +# Add parent directories to path +BACKEND_ROOT = Path(__file__).resolve().parents[1] +TOOLBOX = BACKEND_ROOT / "toolbox" + +if str(BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(BACKEND_ROOT)) +if str(TOOLBOX) not in sys.path: + sys.path.insert(0, str(TOOLBOX)) + + +# ============================================================================ +# Benchmark Fixtures +# ============================================================================ + +@pytest.fixture(scope="session") +def benchmark_fixtures_dir(): + """Path to benchmark fixtures directory""" + return Path(__file__).parent / "fixtures" + + +@pytest.fixture(scope="session") +def small_project_fixture(benchmark_fixtures_dir): + """Small project fixture (~1K LOC)""" + return benchmark_fixtures_dir / "small" + + +@pytest.fixture(scope="session") +def medium_project_fixture(benchmark_fixtures_dir): + """Medium project fixture (~10K LOC)""" + return benchmark_fixtures_dir / "medium" + + +@pytest.fixture(scope="session") +def large_project_fixture(benchmark_fixtures_dir): + """Large project fixture (~100K LOC)""" + return benchmark_fixtures_dir / "large" + + +# ============================================================================ +# pytest-benchmark Configuration +# ============================================================================ + +def pytest_configure(config): + """Configure pytest-benchmark""" + config.addinivalue_line( + "markers", "benchmark: mark test as a benchmark" + ) + + +def pytest_benchmark_group_stats(config, benchmarks, group_by): + """Group benchmark results by category""" + return group_by diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1f3e7b5..03a7307 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,13 +1,14 @@ [project] name = "backend" -version = "0.6.0" +version = "0.7.0" description = "FuzzForge OSS backend" authors = [] readme = "README.md" requires-python = ">=3.11" dependencies = [ "fastapi>=0.116.1", - "prefect>=3.4.18", + "temporalio>=1.6.0", + "boto3>=1.34.0", "pydantic>=2.0.0", "pyyaml>=6.0", "docker>=7.0.0", @@ -21,5 +22,20 @@ dependencies = [ dev = [ "pytest>=8.0.0", "pytest-asyncio>=0.23.0", + "pytest-benchmark>=4.0.0", + "pytest-cov>=5.0.0", + "pytest-xdist>=3.5.0", + "pytest-mock>=3.12.0", "httpx>=0.27.0", + "ruff>=0.1.0", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests", "benchmarks"] +python_files = ["test_*.py", "bench_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +markers = [ + "benchmark: mark test as a benchmark", ] diff --git a/backend/src/api/fuzzing.py b/backend/src/api/fuzzing.py index df4ed86..166319a 100644 --- a/backend/src/api/fuzzing.py +++ b/backend/src/api/fuzzing.py @@ -14,8 +14,8 @@ API endpoints for fuzzing workflow management and real-time monitoring # Additional attribution and requirements are provided in the NOTICE file. import logging -from typing import List, Dict, Any -from fastapi import APIRouter, HTTPException, Depends, WebSocket, WebSocketDisconnect +from typing import List, Dict +from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect from fastapi.responses import StreamingResponse import asyncio import json @@ -25,7 +25,6 @@ from src.models.findings import ( FuzzingStats, CrashReport ) -from src.core.workflow_discovery import WorkflowDiscovery logger = logging.getLogger(__name__) @@ -126,12 +125,13 @@ async def update_fuzzing_stats(run_id: str, stats: FuzzingStats): # Debug: log reception for live instrumentation try: logger.info( - "Received fuzzing stats update: run_id=%s exec=%s eps=%.2f crashes=%s corpus=%s elapsed=%ss", + "Received fuzzing stats update: run_id=%s exec=%s eps=%.2f crashes=%s corpus=%s coverage=%s elapsed=%ss", run_id, stats.executions, stats.executions_per_sec, stats.crashes, stats.corpus_size, + stats.coverage, stats.elapsed_time, ) except Exception: diff --git a/backend/src/api/runs.py b/backend/src/api/runs.py index db63683..727e211 100644 --- a/backend/src/api/runs.py +++ b/backend/src/api/runs.py @@ -14,7 +14,6 @@ API endpoints for workflow run management and findings retrieval # Additional attribution and requirements are provided in the NOTICE file. import logging -from typing import Dict, Any from fastapi import APIRouter, HTTPException, Depends from src.models.findings import WorkflowFindings, WorkflowStatus @@ -24,22 +23,22 @@ logger = logging.getLogger(__name__) router = APIRouter(prefix="/runs", tags=["runs"]) -def get_prefect_manager(): - """Dependency to get the Prefect manager instance""" - from src.main import prefect_mgr - return prefect_mgr +def get_temporal_manager(): + """Dependency to get the Temporal manager instance""" + from src.main import temporal_mgr + return temporal_mgr @router.get("/{run_id}/status", response_model=WorkflowStatus) async def get_run_status( run_id: str, - prefect_mgr=Depends(get_prefect_manager) + temporal_mgr=Depends(get_temporal_manager) ) -> WorkflowStatus: """ Get the current status of a workflow run. Args: - run_id: The flow run ID + run_id: The workflow run ID Returns: Status information including state, timestamps, and completion flags @@ -48,25 +47,23 @@ async def get_run_status( HTTPException: 404 if run not found """ try: - status = await prefect_mgr.get_flow_run_status(run_id) + status = await temporal_mgr.get_workflow_status(run_id) - # Find workflow name from deployment - workflow_name = "unknown" - workflow_deployment_id = status.get("workflow", "") - for name, deployment_id in prefect_mgr.deployments.items(): - if str(deployment_id) == str(workflow_deployment_id): - workflow_name = name - break + # Map Temporal status to response format + workflow_status = status.get("status", "UNKNOWN") + is_completed = workflow_status in ["COMPLETED", "FAILED", "CANCELLED"] + is_failed = workflow_status == "FAILED" + is_running = workflow_status == "RUNNING" return WorkflowStatus( - run_id=status["run_id"], - workflow=workflow_name, - status=status["status"], - is_completed=status["is_completed"], - is_failed=status["is_failed"], - is_running=status["is_running"], - created_at=status["created_at"], - updated_at=status["updated_at"] + run_id=run_id, + workflow="unknown", # Temporal doesn't track workflow name in status + status=workflow_status, + is_completed=is_completed, + is_failed=is_failed, + is_running=is_running, + created_at=status.get("start_time"), + updated_at=status.get("close_time") or status.get("execution_time") ) except Exception as e: @@ -80,13 +77,13 @@ async def get_run_status( @router.get("/{run_id}/findings", response_model=WorkflowFindings) async def get_run_findings( run_id: str, - prefect_mgr=Depends(get_prefect_manager) + temporal_mgr=Depends(get_temporal_manager) ) -> WorkflowFindings: """ Get the findings from a completed workflow run. Args: - run_id: The flow run ID + run_id: The workflow run ID Returns: SARIF-formatted findings from the workflow execution @@ -96,50 +93,46 @@ async def get_run_findings( """ try: # Get run status first - status = await prefect_mgr.get_flow_run_status(run_id) + status = await temporal_mgr.get_workflow_status(run_id) + workflow_status = status.get("status", "UNKNOWN") - if not status["is_completed"]: - if status["is_running"]: + if workflow_status not in ["COMPLETED", "FAILED", "CANCELLED"]: + if workflow_status == "RUNNING": raise HTTPException( status_code=400, - detail=f"Run {run_id} is still running. Current status: {status['status']}" - ) - elif status["is_failed"]: - raise HTTPException( - status_code=400, - detail=f"Run {run_id} failed. Status: {status['status']}" + detail=f"Run {run_id} is still running. Current status: {workflow_status}" ) else: raise HTTPException( status_code=400, - detail=f"Run {run_id} not completed. Status: {status['status']}" + detail=f"Run {run_id} not completed. Status: {workflow_status}" ) - # Get the findings - findings = await prefect_mgr.get_flow_run_findings(run_id) + if workflow_status == "FAILED": + raise HTTPException( + status_code=400, + detail=f"Run {run_id} failed. Status: {workflow_status}" + ) - # Find workflow name - workflow_name = "unknown" - workflow_deployment_id = status.get("workflow", "") - for name, deployment_id in prefect_mgr.deployments.items(): - if str(deployment_id) == str(workflow_deployment_id): - workflow_name = name - break + # Get the workflow result + result = await temporal_mgr.get_workflow_result(run_id) - # Get workflow version if available + # Extract SARIF from result (handle None for backwards compatibility) + if isinstance(result, dict): + sarif = result.get("sarif") or {} + else: + sarif = {} + + # Metadata metadata = { - "completion_time": status["updated_at"], + "completion_time": status.get("close_time"), "workflow_version": "unknown" } - if workflow_name in prefect_mgr.workflows: - workflow_info = prefect_mgr.workflows[workflow_name] - metadata["workflow_version"] = workflow_info.metadata.get("version", "unknown") - return WorkflowFindings( - workflow=workflow_name, + workflow="unknown", run_id=run_id, - sarif=findings, + sarif=sarif, metadata=metadata ) @@ -157,7 +150,7 @@ async def get_run_findings( async def get_workflow_findings( workflow_name: str, run_id: str, - prefect_mgr=Depends(get_prefect_manager) + temporal_mgr=Depends(get_temporal_manager) ) -> WorkflowFindings: """ Get findings for a specific workflow run. @@ -166,7 +159,7 @@ async def get_workflow_findings( Args: workflow_name: Name of the workflow - run_id: The flow run ID + run_id: The workflow run ID Returns: SARIF-formatted findings from the workflow execution @@ -174,11 +167,11 @@ async def get_workflow_findings( Raises: HTTPException: 404 if workflow or run not found, 400 if run not completed """ - if workflow_name not in prefect_mgr.workflows: + if workflow_name not in temporal_mgr.workflows: raise HTTPException( status_code=404, detail=f"Workflow not found: {workflow_name}" ) # Delegate to the main findings endpoint - return await get_run_findings(run_id, prefect_mgr) \ No newline at end of file + return await get_run_findings(run_id, temporal_mgr) diff --git a/backend/src/api/workflows.py b/backend/src/api/workflows.py index dcd504a..513ffea 100644 --- a/backend/src/api/workflows.py +++ b/backend/src/api/workflows.py @@ -15,8 +15,9 @@ API endpoints for workflow management with enhanced error handling import logging import traceback +import tempfile from typing import List, Dict, Any, Optional -from fastapi import APIRouter, HTTPException, Depends +from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form from pathlib import Path from src.models.findings import ( @@ -25,10 +26,20 @@ from src.models.findings import ( WorkflowListItem, RunSubmissionResponse ) -from src.core.workflow_discovery import WorkflowDiscovery +from src.temporal.discovery import WorkflowDiscovery logger = logging.getLogger(__name__) +# Configuration for file uploads +MAX_UPLOAD_SIZE = 10 * 1024 * 1024 * 1024 # 10 GB +ALLOWED_CONTENT_TYPES = [ + "application/gzip", + "application/x-gzip", + "application/x-tar", + "application/x-compressed-tar", + "application/octet-stream", # Generic binary +] + router = APIRouter(prefix="/workflows", tags=["workflows"]) @@ -68,15 +79,15 @@ def create_structured_error_response( return error_response -def get_prefect_manager(): - """Dependency to get the Prefect manager instance""" - from src.main import prefect_mgr - return prefect_mgr +def get_temporal_manager(): + """Dependency to get the Temporal manager instance""" + from src.main import temporal_mgr + return temporal_mgr @router.get("/", response_model=List[WorkflowListItem]) async def list_workflows( - prefect_mgr=Depends(get_prefect_manager) + temporal_mgr=Depends(get_temporal_manager) ) -> List[WorkflowListItem]: """ List all discovered workflows with their metadata. @@ -85,7 +96,7 @@ async def list_workflows( author, and tags. """ workflows = [] - for name, info in prefect_mgr.workflows.items(): + for name, info in temporal_mgr.workflows.items(): workflows.append(WorkflowListItem( name=name, version=info.metadata.get("version", "0.6.0"), @@ -111,7 +122,7 @@ async def get_metadata_schema() -> Dict[str, Any]: @router.get("/{workflow_name}/metadata", response_model=WorkflowMetadata) async def get_workflow_metadata( workflow_name: str, - prefect_mgr=Depends(get_prefect_manager) + temporal_mgr=Depends(get_temporal_manager) ) -> WorkflowMetadata: """ Get complete metadata for a specific workflow. @@ -126,8 +137,8 @@ async def get_workflow_metadata( Raises: HTTPException: 404 if workflow not found """ - if workflow_name not in prefect_mgr.workflows: - available_workflows = list(prefect_mgr.workflows.keys()) + if workflow_name not in temporal_mgr.workflows: + available_workflows = list(temporal_mgr.workflows.keys()) error_response = create_structured_error_response( error_type="WorkflowNotFound", message=f"Workflow '{workflow_name}' not found", @@ -143,7 +154,7 @@ async def get_workflow_metadata( detail=error_response ) - info = prefect_mgr.workflows[workflow_name] + info = temporal_mgr.workflows[workflow_name] metadata = info.metadata return WorkflowMetadata( @@ -154,9 +165,7 @@ async def get_workflow_metadata( tags=metadata.get("tags", []), parameters=metadata.get("parameters", {}), default_parameters=metadata.get("default_parameters", {}), - required_modules=metadata.get("required_modules", []), - supported_volume_modes=metadata.get("supported_volume_modes", ["ro", "rw"]), - has_custom_docker=info.has_docker + required_modules=metadata.get("required_modules", []) ) @@ -164,14 +173,14 @@ async def get_workflow_metadata( async def submit_workflow( workflow_name: str, submission: WorkflowSubmission, - prefect_mgr=Depends(get_prefect_manager) + temporal_mgr=Depends(get_temporal_manager) ) -> RunSubmissionResponse: """ - Submit a workflow for execution with volume mounting. + Submit a workflow for execution. Args: workflow_name: Name of the workflow to execute - submission: Submission parameters including target path and volume mode + submission: Submission parameters including target path and parameters Returns: Run submission response with run_id and initial status @@ -179,8 +188,8 @@ async def submit_workflow( Raises: HTTPException: 404 if workflow not found, 400 for invalid parameters """ - if workflow_name not in prefect_mgr.workflows: - available_workflows = list(prefect_mgr.workflows.keys()) + if workflow_name not in temporal_mgr.workflows: + available_workflows = list(temporal_mgr.workflows.keys()) error_response = create_structured_error_response( error_type="WorkflowNotFound", message=f"Workflow '{workflow_name}' not found", @@ -197,31 +206,36 @@ async def submit_workflow( ) try: - # Convert ResourceLimits to dict if provided - resource_limits_dict = None - if submission.resource_limits: - resource_limits_dict = { - "cpu_limit": submission.resource_limits.cpu_limit, - "memory_limit": submission.resource_limits.memory_limit, - "cpu_request": submission.resource_limits.cpu_request, - "memory_request": submission.resource_limits.memory_request - } + # Upload target file to MinIO and get target_id + target_path = Path(submission.target_path) + if not target_path.exists(): + raise ValueError(f"Target path does not exist: {submission.target_path}") - # Submit the workflow with enhanced parameters - flow_run = await prefect_mgr.submit_workflow( - workflow_name=workflow_name, - target_path=submission.target_path, - volume_mode=submission.volume_mode, - parameters=submission.parameters, - resource_limits=resource_limits_dict, - additional_volumes=submission.additional_volumes, - timeout=submission.timeout + # Upload target (using anonymous user for now) + target_id = await temporal_mgr.upload_target( + file_path=target_path, + user_id="api-user", + metadata={"workflow": workflow_name} ) - run_id = str(flow_run.id) + # Merge default parameters with user parameters + workflow_info = temporal_mgr.workflows[workflow_name] + metadata = workflow_info.metadata or {} + defaults = metadata.get("default_parameters", {}) + user_params = submission.parameters or {} + workflow_params = {**defaults, **user_params} + + # Start workflow execution + handle = await temporal_mgr.run_workflow( + workflow_name=workflow_name, + target_id=target_id, + workflow_params=workflow_params + ) + + run_id = handle.id # Initialize fuzzing tracking if this looks like a fuzzing workflow - workflow_info = prefect_mgr.workflows.get(workflow_name, {}) + workflow_info = temporal_mgr.workflows.get(workflow_name, {}) workflow_tags = workflow_info.metadata.get("tags", []) if hasattr(workflow_info, 'metadata') else [] if "fuzzing" in workflow_tags or "fuzz" in workflow_name.lower(): from src.api.fuzzing import initialize_fuzzing_tracking @@ -229,7 +243,7 @@ async def submit_workflow( return RunSubmissionResponse( run_id=run_id, - status=flow_run.state.name if flow_run.state else "PENDING", + status="RUNNING", workflow=workflow_name, message=f"Workflow '{workflow_name}' submitted successfully" ) @@ -261,17 +275,13 @@ async def submit_workflow( error_type = "WorkflowSubmissionError" # Detect specific error patterns - if "deployment" in error_message.lower(): - error_type = "DeploymentError" - deployment_info = { - "status": "failed", - "error": error_message - } + if "workflow" in error_message.lower() and "not found" in error_message.lower(): + error_type = "WorkflowError" suggestions.extend([ - "Check if Prefect server is running and accessible", - "Verify Docker is running and has sufficient resources", - "Check container image availability", - "Ensure volume paths exist and are accessible" + "Check if Temporal server is running and accessible", + "Verify workflow workers are running", + "Check if workflow is registered with correct vertical", + "Ensure Docker is running and has sufficient resources" ]) elif "volume" in error_message.lower() or "mount" in error_message.lower(): @@ -324,25 +334,200 @@ async def submit_workflow( ) -@router.get("/{workflow_name}/parameters") -async def get_workflow_parameters( +@router.post("/{workflow_name}/upload-and-submit", response_model=RunSubmissionResponse) +async def upload_and_submit_workflow( workflow_name: str, - prefect_mgr=Depends(get_prefect_manager) + file: UploadFile = File(..., description="Target file or tarball to analyze"), + parameters: Optional[str] = Form(None, description="JSON-encoded workflow parameters"), + timeout: Optional[int] = Form(None, description="Timeout in seconds"), + temporal_mgr=Depends(get_temporal_manager) +) -> RunSubmissionResponse: + """ + Upload a target file/tarball and submit workflow for execution. + + This endpoint accepts multipart/form-data uploads and is the recommended + way to submit workflows from remote CLI clients. + + Args: + workflow_name: Name of the workflow to execute + file: Target file or tarball (compressed directory) + parameters: JSON string of workflow parameters (optional) + timeout: Execution timeout in seconds (optional) + + Returns: + Run submission response with run_id and initial status + + Raises: + HTTPException: 404 if workflow not found, 400 for invalid parameters, + 413 if file too large + """ + if workflow_name not in temporal_mgr.workflows: + available_workflows = list(temporal_mgr.workflows.keys()) + error_response = create_structured_error_response( + error_type="WorkflowNotFound", + message=f"Workflow '{workflow_name}' not found", + workflow_name=workflow_name, + suggestions=[ + f"Available workflows: {', '.join(available_workflows)}", + "Use GET /workflows/ to see all available workflows" + ] + ) + raise HTTPException(status_code=404, detail=error_response) + + temp_file_path = None + + try: + # Validate file size + file_size = 0 + chunk_size = 1024 * 1024 # 1MB chunks + + # Create temporary file + temp_fd, temp_file_path = tempfile.mkstemp(suffix=".tar.gz") + + logger.info(f"Receiving file upload for workflow '{workflow_name}': {file.filename}") + + # Stream file to disk + with open(temp_fd, 'wb') as temp_file: + while True: + chunk = await file.read(chunk_size) + if not chunk: + break + + file_size += len(chunk) + + # Check size limit + if file_size > MAX_UPLOAD_SIZE: + raise HTTPException( + status_code=413, + detail=create_structured_error_response( + error_type="FileTooLarge", + message=f"File size exceeds maximum allowed size of {MAX_UPLOAD_SIZE / (1024**3):.1f} GB", + workflow_name=workflow_name, + suggestions=[ + "Reduce the size of your target directory", + "Exclude unnecessary files (build artifacts, dependencies, etc.)", + "Consider splitting into smaller analysis targets" + ] + ) + ) + + temp_file.write(chunk) + + logger.info(f"Received file: {file_size / (1024**2):.2f} MB") + + # Parse parameters + workflow_params = {} + if parameters: + try: + import json + workflow_params = json.loads(parameters) + if not isinstance(workflow_params, dict): + raise ValueError("Parameters must be a JSON object") + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException( + status_code=400, + detail=create_structured_error_response( + error_type="InvalidParameters", + message=f"Invalid parameters JSON: {e}", + workflow_name=workflow_name, + suggestions=["Ensure parameters is valid JSON object"] + ) + ) + + # Upload to MinIO + target_id = await temporal_mgr.upload_target( + file_path=Path(temp_file_path), + user_id="api-user", + metadata={ + "workflow": workflow_name, + "original_filename": file.filename, + "upload_method": "multipart" + } + ) + + logger.info(f"Uploaded to MinIO with target_id: {target_id}") + + # Merge default parameters with user parameters + workflow_info = temporal_mgr.workflows.get(workflow_name) + metadata = workflow_info.metadata or {} + defaults = metadata.get("default_parameters", {}) + workflow_params = {**defaults, **workflow_params} + + # Start workflow execution + handle = await temporal_mgr.run_workflow( + workflow_name=workflow_name, + target_id=target_id, + workflow_params=workflow_params + ) + + run_id = handle.id + + # Initialize fuzzing tracking if needed + workflow_info = temporal_mgr.workflows.get(workflow_name, {}) + workflow_tags = workflow_info.metadata.get("tags", []) if hasattr(workflow_info, 'metadata') else [] + if "fuzzing" in workflow_tags or "fuzz" in workflow_name.lower(): + from src.api.fuzzing import initialize_fuzzing_tracking + initialize_fuzzing_tracking(run_id, workflow_name) + + return RunSubmissionResponse( + run_id=run_id, + status="RUNNING", + workflow=workflow_name, + message=f"Workflow '{workflow_name}' submitted successfully with uploaded target" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to upload and submit workflow '{workflow_name}': {e}") + logger.error(f"Traceback: {traceback.format_exc()}") + + error_response = create_structured_error_response( + error_type="WorkflowSubmissionError", + message=f"Failed to process upload and submit workflow: {str(e)}", + workflow_name=workflow_name, + suggestions=[ + "Check if the uploaded file is a valid tarball", + "Verify MinIO storage is accessible", + "Check backend logs for detailed error information", + "Ensure Temporal workers are running" + ] + ) + + raise HTTPException(status_code=500, detail=error_response) + + finally: + # Cleanup temporary file + if temp_file_path and Path(temp_file_path).exists(): + try: + Path(temp_file_path).unlink() + logger.debug(f"Cleaned up temp file: {temp_file_path}") + except Exception as e: + logger.warning(f"Failed to cleanup temp file {temp_file_path}: {e}") + + +@router.get("/{workflow_name}/worker-info") +async def get_workflow_worker_info( + workflow_name: str, + temporal_mgr=Depends(get_temporal_manager) ) -> Dict[str, Any]: """ - Get the parameters schema for a workflow. + Get worker information for a workflow. + + Returns details about which worker is required to execute this workflow, + including container name, task queue, and vertical. Args: workflow_name: Name of the workflow Returns: - Parameters schema with types, descriptions, and defaults + Worker information including vertical, container name, and task queue Raises: HTTPException: 404 if workflow not found """ - if workflow_name not in prefect_mgr.workflows: - available_workflows = list(prefect_mgr.workflows.keys()) + if workflow_name not in temporal_mgr.workflows: + available_workflows = list(temporal_mgr.workflows.keys()) error_response = create_structured_error_response( error_type="WorkflowNotFound", message=f"Workflow '{workflow_name}' not found", @@ -357,7 +542,71 @@ async def get_workflow_parameters( detail=error_response ) - info = prefect_mgr.workflows[workflow_name] + info = temporal_mgr.workflows[workflow_name] + metadata = info.metadata + + # Extract vertical from metadata + vertical = metadata.get("vertical") + + if not vertical: + error_response = create_structured_error_response( + error_type="MissingVertical", + message=f"Workflow '{workflow_name}' does not specify a vertical in metadata", + workflow_name=workflow_name, + suggestions=[ + "Check workflow metadata.yaml for 'vertical' field", + "Contact workflow author for support" + ] + ) + raise HTTPException( + status_code=500, + detail=error_response + ) + + return { + "workflow": workflow_name, + "vertical": vertical, + "worker_container": f"fuzzforge-worker-{vertical}", + "worker_service": f"worker-{vertical}", + "task_queue": f"{vertical}-queue", + "required": True + } + + +@router.get("/{workflow_name}/parameters") +async def get_workflow_parameters( + workflow_name: str, + temporal_mgr=Depends(get_temporal_manager) +) -> Dict[str, Any]: + """ + Get the parameters schema for a workflow. + + Args: + workflow_name: Name of the workflow + + Returns: + Parameters schema with types, descriptions, and defaults + + Raises: + HTTPException: 404 if workflow not found + """ + if workflow_name not in temporal_mgr.workflows: + available_workflows = list(temporal_mgr.workflows.keys()) + error_response = create_structured_error_response( + error_type="WorkflowNotFound", + message=f"Workflow '{workflow_name}' not found", + workflow_name=workflow_name, + suggestions=[ + f"Available workflows: {', '.join(available_workflows)}", + "Use GET /workflows/ to see all available workflows" + ] + ) + raise HTTPException( + status_code=404, + detail=error_response + ) + + info = temporal_mgr.workflows[workflow_name] metadata = info.metadata # Return parameters with enhanced schema information diff --git a/backend/src/core/prefect_manager.py b/backend/src/core/prefect_manager.py deleted file mode 100644 index 74a0c39..0000000 --- a/backend/src/core/prefect_manager.py +++ /dev/null @@ -1,770 +0,0 @@ -""" -Prefect Manager - Core orchestration for workflow deployment and execution -""" - -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - -import logging -import os -import platform -import re -from pathlib import Path -from typing import Dict, Optional, Any -from prefect import get_client -from prefect.docker import DockerImage -from prefect.client.schemas import FlowRun - -from src.core.workflow_discovery import WorkflowDiscovery, WorkflowInfo - -logger = logging.getLogger(__name__) - - -def get_registry_url(context: str = "default") -> str: - """ - Get the container registry URL to use for a given operation context. - - Goals: - - Work reliably across Linux and macOS Docker Desktop - - Prefer in-network service discovery when running inside containers - - Allow full override via env vars from docker-compose - - Env overrides: - - FUZZFORGE_REGISTRY_PUSH_URL: used for image builds/pushes - - FUZZFORGE_REGISTRY_PULL_URL: used for workers to pull images - """ - # Normalize context - ctx = (context or "default").lower() - - # Always honor explicit overrides first - if ctx in ("push", "build"): - push_url = os.getenv("FUZZFORGE_REGISTRY_PUSH_URL") - if push_url: - logger.debug("Using FUZZFORGE_REGISTRY_PUSH_URL: %s", push_url) - return push_url - # Default to host-published registry for Docker daemon operations - return "localhost:5001" - - if ctx == "pull": - pull_url = os.getenv("FUZZFORGE_REGISTRY_PULL_URL") - if pull_url: - logger.debug("Using FUZZFORGE_REGISTRY_PULL_URL: %s", pull_url) - return pull_url - # Prefect worker pulls via host Docker daemon as well - return "localhost:5001" - - # Default/fallback - return os.getenv("FUZZFORGE_REGISTRY_PULL_URL", os.getenv("FUZZFORGE_REGISTRY_PUSH_URL", "localhost:5001")) - - -def _compose_project_name(default: str = "fuzzforge") -> str: - """Return the docker-compose project name used for network/volume naming. - - Always returns 'fuzzforge' regardless of environment variables. - """ - return "fuzzforge" - - -class PrefectManager: - """ - Manages Prefect deployments and flow runs for discovered workflows. - - This class handles: - - Workflow discovery and registration - - Docker image building through Prefect - - Deployment creation and management - - Flow run submission with volume mounting - - Findings retrieval from completed runs - """ - - def __init__(self, workflows_dir: Path = None): - """ - Initialize the Prefect manager. - - Args: - workflows_dir: Path to the workflows directory (default: toolbox/workflows) - """ - if workflows_dir is None: - workflows_dir = Path("toolbox/workflows") - - self.discovery = WorkflowDiscovery(workflows_dir) - self.workflows: Dict[str, WorkflowInfo] = {} - self.deployments: Dict[str, str] = {} # workflow_name -> deployment_id - - # Security: Define allowed and forbidden paths for host mounting - self.allowed_base_paths = [ - "/tmp", - "/home", - "/Users", # macOS users - "/opt", - "/var/tmp", - "/workspace", # Common container workspace - "/app" # Container application directory (for test projects) - ] - - self.forbidden_paths = [ - "/etc", - "/root", - "/var/run", - "/sys", - "/proc", - "/dev", - "/boot", - "/var/lib/docker", # Critical Docker data - "/var/log", # System logs - "/usr/bin", # System binaries - "/usr/sbin", - "/sbin", - "/bin" - ] - - @staticmethod - def _parse_memory_to_bytes(memory_str: str) -> int: - """ - Parse memory string (like '512Mi', '1Gi') to bytes. - - Args: - memory_str: Memory string with unit suffix - - Returns: - Memory in bytes - - Raises: - ValueError: If format is invalid - """ - if not memory_str: - return 0 - - match = re.match(r'^(\d+(?:\.\d+)?)\s*([GMK]i?)$', memory_str.strip()) - if not match: - raise ValueError(f"Invalid memory format: {memory_str}. Expected format like '512Mi', '1Gi'") - - value, unit = match.groups() - value = float(value) - - # Convert to bytes based on unit (binary units: Ki, Mi, Gi) - if unit in ['K', 'Ki']: - multiplier = 1024 - elif unit in ['M', 'Mi']: - multiplier = 1024 * 1024 - elif unit in ['G', 'Gi']: - multiplier = 1024 * 1024 * 1024 - else: - raise ValueError(f"Unsupported memory unit: {unit}") - - return int(value * multiplier) - - @staticmethod - def _parse_cpu_to_millicores(cpu_str: str) -> int: - """ - Parse CPU string (like '500m', '1', '2.5') to millicores. - - Args: - cpu_str: CPU string - - Returns: - CPU in millicores (1 core = 1000 millicores) - - Raises: - ValueError: If format is invalid - """ - if not cpu_str: - return 0 - - cpu_str = cpu_str.strip() - - # Handle millicores format (e.g., '500m') - if cpu_str.endswith('m'): - try: - return int(cpu_str[:-1]) - except ValueError: - raise ValueError(f"Invalid CPU format: {cpu_str}") - - # Handle core format (e.g., '1', '2.5') - try: - cores = float(cpu_str) - return int(cores * 1000) # Convert to millicores - except ValueError: - raise ValueError(f"Invalid CPU format: {cpu_str}") - - def _extract_resource_requirements(self, workflow_info: WorkflowInfo) -> Dict[str, str]: - """ - Extract resource requirements from workflow metadata. - - Args: - workflow_info: Workflow information with metadata - - Returns: - Dictionary with resource requirements in Docker format - """ - metadata = workflow_info.metadata - requirements = metadata.get("requirements", {}) - resources = requirements.get("resources", {}) - - resource_config = {} - - # Extract memory requirement - memory = resources.get("memory") - if memory: - try: - # Validate memory format and store original string for Docker - self._parse_memory_to_bytes(memory) - resource_config["memory"] = memory - except ValueError as e: - logger.warning(f"Invalid memory requirement in {workflow_info.name}: {e}") - - # Extract CPU requirement - cpu = resources.get("cpu") - if cpu: - try: - # Validate CPU format and store original string for Docker - self._parse_cpu_to_millicores(cpu) - resource_config["cpus"] = cpu - except ValueError as e: - logger.warning(f"Invalid CPU requirement in {workflow_info.name}: {e}") - - # Extract timeout - timeout = resources.get("timeout") - if timeout and isinstance(timeout, int): - resource_config["timeout"] = str(timeout) - - return resource_config - - async def initialize(self): - """ - Initialize the manager by discovering and deploying all workflows. - - This method: - 1. Discovers all valid workflows in the workflows directory - 2. Validates their metadata - 3. Deploys each workflow to Prefect with Docker images - """ - try: - # Discover workflows - self.workflows = await self.discovery.discover_workflows() - - if not self.workflows: - logger.warning("No workflows discovered") - return - - logger.info(f"Discovered {len(self.workflows)} workflows: {list(self.workflows.keys())}") - - # Deploy each workflow - for name, info in self.workflows.items(): - try: - await self._deploy_workflow(name, info) - except Exception as e: - logger.error(f"Failed to deploy workflow '{name}': {e}") - - except Exception as e: - logger.error(f"Failed to initialize Prefect manager: {e}") - raise - - async def _deploy_workflow(self, name: str, info: WorkflowInfo): - """ - Deploy a single workflow to Prefect with Docker image. - - Args: - name: Workflow name - info: Workflow information including metadata and paths - """ - logger.info(f"Deploying workflow '{name}'...") - - # Get the flow function from registry - flow_func = self.discovery.get_flow_function(name) - if not flow_func: - logger.error( - f"Failed to get flow function for '{name}' from registry. " - f"Ensure the workflow is properly registered in toolbox/workflows/registry.py" - ) - return - - # Use the mandatory Dockerfile with absolute paths for Docker Compose - # Get absolute paths for build context and dockerfile - toolbox_path = info.path.parent.parent.resolve() - dockerfile_abs_path = info.dockerfile.resolve() - - # Calculate relative dockerfile path from toolbox context - try: - dockerfile_rel_path = dockerfile_abs_path.relative_to(toolbox_path) - except ValueError: - # If relative path fails, use the workflow-specific path - dockerfile_rel_path = Path("workflows") / name / "Dockerfile" - - # Determine deployment strategy based on Dockerfile presence - base_image = "prefecthq/prefect:3-python3.11" - has_custom_dockerfile = info.has_docker and info.dockerfile.exists() - - logger.info(f"=== DEPLOYMENT DEBUG for '{name}' ===") - logger.info(f"info.has_docker: {info.has_docker}") - logger.info(f"info.dockerfile: {info.dockerfile}") - logger.info(f"info.dockerfile.exists(): {info.dockerfile.exists()}") - logger.info(f"has_custom_dockerfile: {has_custom_dockerfile}") - logger.info(f"toolbox_path: {toolbox_path}") - logger.info(f"dockerfile_rel_path: {dockerfile_rel_path}") - - if has_custom_dockerfile: - logger.info(f"Workflow '{name}' has custom Dockerfile - building custom image") - # Decide whether to use registry or keep images local to host engine - import os - # Default to using the local registry; set FUZZFORGE_USE_REGISTRY=false to bypass (not recommended) - use_registry = os.getenv("FUZZFORGE_USE_REGISTRY", "true").lower() == "true" - - if use_registry: - registry_url = get_registry_url(context="push") - image_spec = DockerImage( - name=f"{registry_url}/fuzzforge/{name}", - tag="latest", - dockerfile=str(dockerfile_rel_path), - context=str(toolbox_path) - ) - deploy_image = f"{registry_url}/fuzzforge/{name}:latest" - build_custom = True - push_custom = True - logger.info(f"Using registry: {registry_url} for '{name}'") - else: - # Single-host mode: build into host engine cache; no push required - image_spec = DockerImage( - name=f"fuzzforge/{name}", - tag="latest", - dockerfile=str(dockerfile_rel_path), - context=str(toolbox_path) - ) - deploy_image = f"fuzzforge/{name}:latest" - build_custom = True - push_custom = False - logger.info("Using single-host image (no registry push): %s", deploy_image) - else: - logger.info(f"Workflow '{name}' using base image - no custom dependencies needed") - deploy_image = base_image - build_custom = False - push_custom = False - - # Pre-validate registry connectivity when pushing - if push_custom: - try: - from .setup import validate_registry_connectivity - await validate_registry_connectivity(registry_url) - logger.info(f"Registry connectivity validated for {registry_url}") - except Exception as e: - logger.error(f"Registry connectivity validation failed for {registry_url}: {e}") - raise RuntimeError(f"Cannot deploy workflow '{name}': Registry {registry_url} is not accessible. {e}") - - # Deploy the workflow - try: - # Ensure any previous deployment is removed so job variables are updated - try: - async with get_client() as client: - existing = await client.read_deployment_by_name( - f"{name}/{name}-deployment" - ) - if existing: - logger.info(f"Removing existing deployment for '{name}' to refresh settings...") - await client.delete_deployment(existing.id) - except Exception: - # If not found or deletion fails, continue with deployment - pass - - # Extract resource requirements from metadata - workflow_resource_requirements = self._extract_resource_requirements(info) - logger.info(f"Workflow '{name}' resource requirements: {workflow_resource_requirements}") - - # Build job variables with resource requirements - job_variables = { - "image": deploy_image, # Use the worker-accessible registry name - "volumes": [], # Populated at run submission with toolbox mount - "env": { - "PYTHONPATH": "/opt/prefect/toolbox:/opt/prefect", - "WORKFLOW_NAME": name - } - } - - # Add resource requirements to job variables if present - if workflow_resource_requirements: - job_variables["resources"] = workflow_resource_requirements - - # Prepare deployment parameters - deploy_params = { - "name": f"{name}-deployment", - "work_pool_name": "docker-pool", - "image": image_spec if has_custom_dockerfile else deploy_image, - "push": push_custom, - "build": build_custom, - "job_variables": job_variables - } - - deployment = await flow_func.deploy(**deploy_params) - - self.deployments[name] = str(deployment.id) if hasattr(deployment, 'id') else name - logger.info(f"Successfully deployed workflow '{name}'") - - except Exception as e: - # Enhanced error reporting with more context - import traceback - logger.error(f"Failed to deploy workflow '{name}': {e}") - logger.error(f"Deployment traceback: {traceback.format_exc()}") - - # Try to capture Docker-specific context - error_context = { - "workflow_name": name, - "has_dockerfile": has_custom_dockerfile, - "image_name": deploy_image if 'deploy_image' in locals() else "unknown", - "registry_url": registry_url if 'registry_url' in locals() else "unknown", - "error_type": type(e).__name__, - "error_message": str(e) - } - - # Check for specific error patterns with detailed categorization - error_msg_lower = str(e).lower() - if "registry" in error_msg_lower and ("no such host" in error_msg_lower or "connection" in error_msg_lower): - error_context["category"] = "registry_connectivity_error" - error_context["solution"] = f"Cannot reach registry at {error_context['registry_url']}. Check Docker network and registry service." - elif "docker" in error_msg_lower: - error_context["category"] = "docker_error" - if "build" in error_msg_lower: - error_context["subcategory"] = "image_build_failed" - error_context["solution"] = "Check Dockerfile syntax and dependencies." - elif "pull" in error_msg_lower: - error_context["subcategory"] = "image_pull_failed" - error_context["solution"] = "Check if image exists in registry and network connectivity." - elif "push" in error_msg_lower: - error_context["subcategory"] = "image_push_failed" - error_context["solution"] = f"Check registry connectivity and push permissions to {error_context['registry_url']}." - elif "registry" in error_msg_lower: - error_context["category"] = "registry_error" - error_context["solution"] = "Check registry configuration and accessibility." - elif "prefect" in error_msg_lower: - error_context["category"] = "prefect_error" - error_context["solution"] = "Check Prefect server connectivity and deployment configuration." - else: - error_context["category"] = "unknown_deployment_error" - error_context["solution"] = "Check logs for more specific error details." - - logger.error(f"Deployment error context: {error_context}") - - # Raise enhanced exception with context - enhanced_error = Exception(f"Deployment failed for workflow '{name}': {str(e)} | Context: {error_context}") - enhanced_error.original_error = e - enhanced_error.context = error_context - raise enhanced_error - - async def submit_workflow( - self, - workflow_name: str, - target_path: str, - volume_mode: str = "ro", - parameters: Dict[str, Any] = None, - resource_limits: Dict[str, str] = None, - additional_volumes: list = None, - timeout: int = None - ) -> FlowRun: - """ - Submit a workflow for execution with volume mounting. - - Args: - workflow_name: Name of the workflow to execute - target_path: Host path to mount as volume - volume_mode: Volume mount mode ("ro" for read-only, "rw" for read-write) - parameters: Workflow-specific parameters - resource_limits: CPU/memory limits for container - additional_volumes: List of additional volume mounts - timeout: Timeout in seconds - - Returns: - FlowRun object with run information - - Raises: - ValueError: If workflow not found or volume mode not supported - """ - if workflow_name not in self.workflows: - raise ValueError(f"Unknown workflow: {workflow_name}") - - # Validate volume mode - workflow_info = self.workflows[workflow_name] - supported_modes = workflow_info.metadata.get("supported_volume_modes", ["ro", "rw"]) - - if volume_mode not in supported_modes: - raise ValueError( - f"Workflow '{workflow_name}' doesn't support volume mode '{volume_mode}'. " - f"Supported modes: {supported_modes}" - ) - - # Validate target path with security checks - self._validate_target_path(target_path) - - # Validate additional volumes if provided - if additional_volumes: - for volume in additional_volumes: - self._validate_target_path(volume.host_path) - - async with get_client() as client: - # Get the deployment, auto-redeploy once if missing - try: - deployment = await client.read_deployment_by_name( - f"{workflow_name}/{workflow_name}-deployment" - ) - except Exception as e: - import traceback - logger.error(f"Failed to find deployment for workflow '{workflow_name}': {e}") - logger.error(f"Deployment lookup traceback: {traceback.format_exc()}") - - # Attempt a one-time auto-deploy to recover from startup races - try: - logger.info(f"Auto-deploying missing workflow '{workflow_name}' and retrying...") - await self._deploy_workflow(workflow_name, workflow_info) - deployment = await client.read_deployment_by_name( - f"{workflow_name}/{workflow_name}-deployment" - ) - except Exception as redeploy_exc: - # Enhanced error with context - error_context = { - "workflow_name": workflow_name, - "error_type": type(e).__name__, - "error_message": str(e), - "redeploy_error": str(redeploy_exc), - "available_deployments": list(self.deployments.keys()), - } - enhanced_error = ValueError( - f"Deployment not found and redeploy failed for workflow '{workflow_name}': {e} | Context: {error_context}" - ) - enhanced_error.context = error_context - raise enhanced_error - - # Determine the Docker Compose network name and volume names - # Hardcoded to 'fuzzforge' to avoid directory name dependencies - import os - compose_project = "fuzzforge" - docker_network = "fuzzforge_default" - - # Build volume mounts - # Add toolbox volume mount for workflow code access - backend_toolbox_path = "/app/toolbox" # Path in backend container - - # Hardcoded volume names - prefect_storage_volume = "fuzzforge_prefect_storage" - toolbox_code_volume = "fuzzforge_toolbox_code" - - volumes = [ - f"{target_path}:/workspace:{volume_mode}", - f"{prefect_storage_volume}:/prefect-storage", # Shared storage for results - f"{toolbox_code_volume}:/opt/prefect/toolbox:ro" # Mount workflow code - ] - - # Add additional volumes if provided - if additional_volumes: - for volume in additional_volumes: - volume_spec = f"{volume.host_path}:{volume.container_path}:{volume.mode}" - volumes.append(volume_spec) - - # Build environment variables - env_vars = { - "PREFECT_API_URL": "http://prefect-server:4200/api", # Use internal network hostname - "PREFECT_LOGGING_LEVEL": "INFO", - "PREFECT_LOCAL_STORAGE_PATH": "/prefect-storage", # Use shared storage - "PREFECT_RESULTS_PERSIST_BY_DEFAULT": "true", # Enable result persistence - "PREFECT_DEFAULT_RESULT_STORAGE_BLOCK": "local-file-system/fuzzforge-results", # Use our storage block - "WORKSPACE_PATH": "/workspace", - "VOLUME_MODE": volume_mode, - "WORKFLOW_NAME": workflow_name - } - - # Add additional volume paths to environment for easy access - if additional_volumes: - for i, volume in enumerate(additional_volumes): - env_vars[f"ADDITIONAL_VOLUME_{i}_PATH"] = volume.container_path - - # Determine which image to use based on workflow configuration - workflow_info = self.workflows[workflow_name] - has_custom_dockerfile = workflow_info.has_docker and workflow_info.dockerfile.exists() - # Use pull context for worker to pull from registry - registry_url = get_registry_url(context="pull") - workflow_image = f"{registry_url}/fuzzforge/{workflow_name}:latest" if has_custom_dockerfile else "prefecthq/prefect:3-python3.11" - logger.debug(f"Worker will pull image: {workflow_image} (Registry: {registry_url})") - - # Configure job variables with volume mounting and network access - job_variables = { - # Use custom image if available, otherwise base Prefect image - "image": workflow_image, - "volumes": volumes, - "networks": [docker_network], # Connect to Docker Compose network - "env": { - **env_vars, - "PYTHONPATH": "/opt/prefect/toolbox:/opt/prefect/toolbox/workflows", - "WORKFLOW_NAME": workflow_name - } - } - - # Apply resource requirements from workflow metadata and user overrides - workflow_resource_requirements = self._extract_resource_requirements(workflow_info) - final_resource_config = {} - - # Start with workflow requirements as base - if workflow_resource_requirements: - final_resource_config.update(workflow_resource_requirements) - - # Apply user-provided resource limits (overrides workflow defaults) - if resource_limits: - user_resource_config = {} - if resource_limits.get("cpu_limit"): - user_resource_config["cpus"] = resource_limits["cpu_limit"] - if resource_limits.get("memory_limit"): - user_resource_config["memory"] = resource_limits["memory_limit"] - # Note: cpu_request and memory_request are not directly supported by Docker - # but could be used for Kubernetes in the future - - # User overrides take precedence - final_resource_config.update(user_resource_config) - - # Apply final resource configuration - if final_resource_config: - job_variables["resources"] = final_resource_config - logger.info(f"Applied resource limits: {final_resource_config}") - - # Merge parameters with defaults from metadata - default_params = workflow_info.metadata.get("default_parameters", {}) - final_params = {**default_params, **(parameters or {})} - - # Set flow parameters that match the flow signature - final_params["target_path"] = "/workspace" # Container path where volume is mounted - final_params["volume_mode"] = volume_mode - - # Create and submit the flow run - # Pass job_variables to ensure network, volumes, and environment are configured - logger.info(f"Submitting flow with job_variables: {job_variables}") - logger.info(f"Submitting flow with parameters: {final_params}") - - # Prepare flow run creation parameters - flow_run_params = { - "deployment_id": deployment.id, - "parameters": final_params, - "job_variables": job_variables - } - - # Note: Timeout is handled through workflow-level configuration - # Additional timeout configuration can be added to deployment metadata if needed - - flow_run = await client.create_flow_run_from_deployment(**flow_run_params) - - logger.info( - f"Submitted workflow '{workflow_name}' with run_id: {flow_run.id}, " - f"target: {target_path}, mode: {volume_mode}" - ) - - return flow_run - - async def get_flow_run_findings(self, run_id: str) -> Dict[str, Any]: - """ - Retrieve findings from a completed flow run. - - Args: - run_id: The flow run ID - - Returns: - Dictionary containing SARIF-formatted findings - - Raises: - ValueError: If run not completed or not found - """ - async with get_client() as client: - flow_run = await client.read_flow_run(run_id) - - if not flow_run.state.is_completed(): - raise ValueError( - f"Flow run {run_id} not completed. Current status: {flow_run.state.name}" - ) - - # Get the findings from the flow run result - try: - findings = await flow_run.state.result() - return findings - except Exception as e: - logger.error(f"Failed to retrieve findings for run {run_id}: {e}") - raise ValueError(f"Failed to retrieve findings: {e}") - - async def get_flow_run_status(self, run_id: str) -> Dict[str, Any]: - """ - Get the current status of a flow run. - - Args: - run_id: The flow run ID - - Returns: - Dictionary with status information - """ - async with get_client() as client: - flow_run = await client.read_flow_run(run_id) - - return { - "run_id": str(flow_run.id), - "workflow": flow_run.deployment_id, - "status": flow_run.state.name, - "is_completed": flow_run.state.is_completed(), - "is_failed": flow_run.state.is_failed(), - "is_running": flow_run.state.is_running(), - "created_at": flow_run.created, - "updated_at": flow_run.updated - } - - def _validate_target_path(self, target_path: str) -> None: - """ - Validate target path for security before mounting as volume. - - Args: - target_path: Host path to validate - - Raises: - ValueError: If path is not allowed for security reasons - """ - target = Path(target_path) - - # Path must be absolute - if not target.is_absolute(): - raise ValueError(f"Target path must be absolute: {target_path}") - - # Resolve path to handle symlinks and relative components - try: - resolved_path = target.resolve() - except (OSError, RuntimeError) as e: - raise ValueError(f"Cannot resolve target path: {target_path} - {e}") - - resolved_str = str(resolved_path) - - # Check against forbidden paths first (more restrictive) - for forbidden in self.forbidden_paths: - if resolved_str.startswith(forbidden): - raise ValueError( - f"Access denied: Path '{target_path}' resolves to forbidden directory '{forbidden}'. " - f"This path contains sensitive system files and cannot be mounted." - ) - - # Check if path starts with any allowed base path - path_allowed = False - for allowed in self.allowed_base_paths: - if resolved_str.startswith(allowed): - path_allowed = True - break - - if not path_allowed: - allowed_list = ", ".join(self.allowed_base_paths) - raise ValueError( - f"Access denied: Path '{target_path}' is not in allowed directories. " - f"Allowed base paths: {allowed_list}" - ) - - # Additional security checks - if resolved_str == "/": - raise ValueError("Cannot mount root filesystem") - - # Warn if path doesn't exist (but don't block - it might be created later) - if not resolved_path.exists(): - logger.warning(f"Target path does not exist: {target_path}") - - logger.info(f"Path validation passed for: {target_path} -> {resolved_str}") diff --git a/backend/src/core/setup.py b/backend/src/core/setup.py index 16ed60e..97b3a46 100644 --- a/backend/src/core/setup.py +++ b/backend/src/core/setup.py @@ -1,5 +1,5 @@ """ -Setup utilities for Prefect infrastructure +Setup utilities for FuzzForge infrastructure """ # Copyright (c) 2025 FuzzingLabs @@ -14,364 +14,21 @@ Setup utilities for Prefect infrastructure # Additional attribution and requirements are provided in the NOTICE file. import logging -from prefect import get_client -from prefect.client.schemas.actions import WorkPoolCreate -from prefect.client.schemas.objects import WorkPool -from .prefect_manager import get_registry_url logger = logging.getLogger(__name__) -async def setup_docker_pool(): - """ - Create or update the Docker work pool for container execution. - - This work pool is configured to: - - Connect to the local Docker daemon - - Support volume mounting at runtime - - Clean up containers after execution - - Use bridge networking by default - """ - import os - - async with get_client() as client: - pool_name = "docker-pool" - - # Add force recreation flag for debugging fresh install issues - force_recreate = os.getenv('FORCE_RECREATE_WORK_POOL', 'false').lower() == 'true' - debug_setup = os.getenv('DEBUG_WORK_POOL_SETUP', 'false').lower() == 'true' - - if force_recreate: - logger.warning(f"FORCE_RECREATE_WORK_POOL=true - Will recreate work pool regardless of existing configuration") - if debug_setup: - logger.warning(f"DEBUG_WORK_POOL_SETUP=true - Enhanced logging enabled") - # Temporarily set logging level to DEBUG for this function - original_level = logger.level - logger.setLevel(logging.DEBUG) - - try: - # Check if pool already exists and supports custom images - existing_pools = await client.read_work_pools() - existing_pool = None - for pool in existing_pools: - if pool.name == pool_name: - existing_pool = pool - break - - if existing_pool and not force_recreate: - logger.info(f"Found existing work pool '{pool_name}' - validating configuration...") - - # Check if the existing pool has the correct configuration - base_template = existing_pool.base_job_template or {} - logger.debug(f"Base template keys: {list(base_template.keys())}") - - job_config = base_template.get("job_configuration", {}) - logger.debug(f"Job config keys: {list(job_config.keys())}") - - image_config = job_config.get("image", "") - has_image_variable = "{{ image }}" in str(image_config) - logger.debug(f"Image config: '{image_config}' -> has_image_variable: {has_image_variable}") - - # Check if volume defaults include toolbox mount - variables = base_template.get("variables", {}) - properties = variables.get("properties", {}) - volume_config = properties.get("volumes", {}) - volume_defaults = volume_config.get("default", []) - has_toolbox_volume = any("toolbox_code" in str(vol) for vol in volume_defaults) if volume_defaults else False - logger.debug(f"Volume defaults: {volume_defaults}") - logger.debug(f"Has toolbox volume: {has_toolbox_volume}") - - # Check if environment defaults include required settings - env_config = properties.get("env", {}) - env_defaults = env_config.get("default", {}) - has_api_url = "PREFECT_API_URL" in env_defaults - has_storage_path = "PREFECT_LOCAL_STORAGE_PATH" in env_defaults - has_results_persist = "PREFECT_RESULTS_PERSIST_BY_DEFAULT" in env_defaults - has_required_env = has_api_url and has_storage_path and has_results_persist - logger.debug(f"Environment defaults: {env_defaults}") - logger.debug(f"Has API URL: {has_api_url}, Has storage path: {has_storage_path}, Has results persist: {has_results_persist}") - logger.debug(f"Has required env: {has_required_env}") - - # Log the full validation result - logger.info(f"Work pool validation - Image: {has_image_variable}, Toolbox: {has_toolbox_volume}, Environment: {has_required_env}") - - if has_image_variable and has_toolbox_volume and has_required_env: - logger.info(f"Docker work pool '{pool_name}' already exists with correct configuration") - return - else: - reasons = [] - if not has_image_variable: - reasons.append("missing image template") - if not has_toolbox_volume: - reasons.append("missing toolbox volume mount") - if not has_required_env: - if not has_api_url: - reasons.append("missing PREFECT_API_URL") - if not has_storage_path: - reasons.append("missing PREFECT_LOCAL_STORAGE_PATH") - if not has_results_persist: - reasons.append("missing PREFECT_RESULTS_PERSIST_BY_DEFAULT") - - logger.warning(f"Docker work pool '{pool_name}' exists but lacks: {', '.join(reasons)}. Recreating...") - # Delete the old pool and recreate it - try: - await client.delete_work_pool(pool_name) - logger.info(f"Deleted old work pool '{pool_name}'") - except Exception as e: - logger.warning(f"Failed to delete old work pool: {e}") - elif force_recreate and existing_pool: - logger.warning(f"Force recreation enabled - deleting existing work pool '{pool_name}'") - try: - await client.delete_work_pool(pool_name) - logger.info(f"Deleted existing work pool for force recreation") - except Exception as e: - logger.warning(f"Failed to delete work pool for force recreation: {e}") - - logger.info(f"Creating Docker work pool '{pool_name}' with custom image support...") - - # Create the work pool with proper Docker configuration - work_pool = WorkPoolCreate( - name=pool_name, - type="docker", - description="Docker work pool for FuzzForge workflows with custom image support", - base_job_template={ - "job_configuration": { - "image": "{{ image }}", # Template variable for custom images - "volumes": "{{ volumes }}", # List of volume mounts - "env": "{{ env }}", # Environment variables - "networks": "{{ networks }}", # Docker networks - "stream_output": True, - "auto_remove": True, - "privileged": False, - "network_mode": None, # Use networks instead - "labels": {}, - "command": None # Let the image's CMD/ENTRYPOINT run - }, - "variables": { - "type": "object", - "properties": { - "image": { - "type": "string", - "title": "Docker Image", - "default": "prefecthq/prefect:3-python3.11", - "description": "Docker image for the flow run" - }, - "volumes": { - "type": "array", - "title": "Volume Mounts", - "default": [ - "fuzzforge_prefect_storage:/prefect-storage", - "fuzzforge_toolbox_code:/opt/prefect/toolbox:ro" - ], - "description": "Volume mounts in format 'host:container:mode'", - "items": { - "type": "string" - } - }, - "networks": { - "type": "array", - "title": "Docker Networks", - "default": ["fuzzforge_default"], - "description": "Docker networks to connect container to", - "items": { - "type": "string" - } - }, - "env": { - "type": "object", - "title": "Environment Variables", - "default": { - "PREFECT_API_URL": "http://prefect-server:4200/api", - "PREFECT_LOCAL_STORAGE_PATH": "/prefect-storage", - "PREFECT_RESULTS_PERSIST_BY_DEFAULT": "true" - }, - "description": "Environment variables for the container", - "additionalProperties": { - "type": "string" - } - } - } - } - } - ) - - await client.create_work_pool(work_pool) - logger.info(f"Created Docker work pool '{pool_name}'") - - except Exception as e: - logger.error(f"Failed to setup Docker work pool: {e}") - raise - finally: - # Restore original logging level if debug mode was enabled - if debug_setup and 'original_level' in locals(): - logger.setLevel(original_level) - - -def get_actual_compose_project_name(): - """ - Return the hardcoded compose project name for FuzzForge. - - Always returns 'fuzzforge' as per system requirements. - """ - logger.info("Using hardcoded compose project name: fuzzforge") - return "fuzzforge" - - async def setup_result_storage(): """ - Create or update Prefect result storage block for findings persistence. + Setup result storage (MinIO). - This sets up a LocalFileSystem storage block pointing to the shared - /prefect-storage volume for result persistence. + MinIO is used for both target upload and result storage. + This is a placeholder for any MinIO-specific setup if needed. """ - from prefect.filesystems import LocalFileSystem - - storage_name = "fuzzforge-results" - - try: - # Create the storage block, overwrite if it exists - logger.info(f"Setting up storage block '{storage_name}'...") - storage = LocalFileSystem(basepath="/prefect-storage") - - block_doc_id = await storage.save(name=storage_name, overwrite=True) - logger.info(f"Storage block '{storage_name}' configured successfully") - return str(block_doc_id) - - except Exception as e: - logger.error(f"Failed to setup result storage: {e}") - # Don't raise the exception - continue without storage block - logger.warning("Continuing without result storage block - findings may not persist") - return None - - -async def validate_docker_connection(): - """ - Validate that Docker is accessible and running. - - Note: In containerized deployments with Docker socket proxy, - the backend doesn't need direct Docker access. - - Raises: - RuntimeError: If Docker is not accessible - """ - import os - - # Skip Docker validation if running in container without socket access - if os.path.exists("/.dockerenv") and not os.path.exists("/var/run/docker.sock"): - logger.info("Running in container without Docker socket - skipping Docker validation") - return - - try: - import docker - client = docker.from_env() - client.ping() - logger.info("Docker connection validated") - except Exception as e: - logger.error(f"Docker is not accessible: {e}") - raise RuntimeError( - "Docker is not running or not accessible. " - "Please ensure Docker is installed and running." - ) - - -async def validate_registry_connectivity(registry_url: str = None): - """ - Validate that the Docker registry is accessible. - - Args: - registry_url: URL of the Docker registry to validate (auto-detected if None) - - Raises: - RuntimeError: If registry is not accessible - """ - # Resolve a reachable test URL from within this process - if registry_url is None: - # If not specified, prefer internal service name in containers, host port on host - import os - if os.path.exists('/.dockerenv'): - registry_url = "registry:5000" - else: - registry_url = "localhost:5001" - - # If we're running inside a container and asked to probe localhost:PORT, - # the probe would hit the container, not the host. Use host.docker.internal instead. - import os - try: - host_part, port_part = registry_url.split(":", 1) - except ValueError: - host_part, port_part = registry_url, "80" - - if os.path.exists('/.dockerenv') and host_part in ("localhost", "127.0.0.1"): - test_host = "host.docker.internal" - else: - test_host = host_part - test_url = f"http://{test_host}:{port_part}/v2/" - - import aiohttp - import asyncio - - logger.info(f"Validating registry connectivity to {registry_url}...") - - try: - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session: - async with session.get(test_url) as response: - if response.status == 200: - logger.info(f"Registry at {registry_url} is accessible (tested via {test_host})") - return - else: - raise RuntimeError(f"Registry returned status {response.status}") - except asyncio.TimeoutError: - raise RuntimeError(f"Registry at {registry_url} is not responding (timeout)") - except aiohttp.ClientError as e: - raise RuntimeError(f"Registry at {registry_url} is not accessible: {e}") - except Exception as e: - raise RuntimeError(f"Failed to validate registry connectivity: {e}") - - -async def validate_docker_network(network_name: str): - """ - Validate that the specified Docker network exists. - - Args: - network_name: Name of the Docker network to validate - - Raises: - RuntimeError: If network doesn't exist - """ - import os - - # Skip network validation if running in container without Docker socket - if os.path.exists("/.dockerenv") and not os.path.exists("/var/run/docker.sock"): - logger.info("Running in container without Docker socket - skipping network validation") - return - - try: - import docker - client = docker.from_env() - - # List all networks - networks = client.networks.list(names=[network_name]) - - if not networks: - # Try to find networks with similar names - all_networks = client.networks.list() - similar_networks = [n.name for n in all_networks if "fuzzforge" in n.name.lower()] - - error_msg = f"Docker network '{network_name}' not found." - if similar_networks: - error_msg += f" Available networks: {similar_networks}" - else: - error_msg += " Please ensure Docker Compose is running." - - raise RuntimeError(error_msg) - - logger.info(f"Docker network '{network_name}' validated") - - except Exception as e: - if isinstance(e, RuntimeError): - raise - logger.error(f"Network validation failed: {e}") - raise RuntimeError(f"Failed to validate Docker network: {e}") + logger.info("Result storage (MinIO) configured") + # MinIO is configured via environment variables in docker-compose + # No additional setup needed here + return True async def validate_infrastructure(): @@ -382,21 +39,7 @@ async def validate_infrastructure(): """ logger.info("Validating infrastructure...") - # Validate Docker connection - await validate_docker_connection() - - # Validate registry connectivity for custom image building - await validate_registry_connectivity() - - # Validate network (hardcoded to avoid directory name dependencies) - import os - compose_project = "fuzzforge" - docker_network = "fuzzforge_default" - - try: - await validate_docker_network(docker_network) - except RuntimeError as e: - logger.warning(f"Network validation failed: {e}") - logger.warning("Workflows may not be able to connect to Prefect services") + # Setup storage (MinIO) + await setup_result_storage() logger.info("Infrastructure validation completed") diff --git a/backend/src/core/workflow_discovery.py b/backend/src/core/workflow_discovery.py deleted file mode 100644 index e348524..0000000 --- a/backend/src/core/workflow_discovery.py +++ /dev/null @@ -1,459 +0,0 @@ -""" -Workflow Discovery - Registry-based discovery and loading of workflows -""" - -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - -import logging -import yaml -from pathlib import Path -from typing import Dict, Optional, Any, Callable -from pydantic import BaseModel, Field, ConfigDict - -logger = logging.getLogger(__name__) - - -class WorkflowInfo(BaseModel): - """Information about a discovered workflow""" - name: str = Field(..., description="Workflow name") - path: Path = Field(..., description="Path to workflow directory") - workflow_file: Path = Field(..., description="Path to workflow.py file") - dockerfile: Path = Field(..., description="Path to Dockerfile") - has_docker: bool = Field(..., description="Whether workflow has custom Dockerfile") - metadata: Dict[str, Any] = Field(..., description="Workflow metadata from YAML") - flow_function_name: str = Field(default="main_flow", description="Name of the flow function") - - model_config = ConfigDict(arbitrary_types_allowed=True) - - -class WorkflowDiscovery: - """ - Discovers workflows from the filesystem and validates them against the registry. - - This system: - 1. Scans for workflows with metadata.yaml files - 2. Cross-references them with the manual registry - 3. Provides registry-based flow functions for deployment - - Workflows must have: - - workflow.py: Contains the Prefect flow - - metadata.yaml: Mandatory metadata file - - Entry in toolbox/workflows/registry.py: Manual registration - - Dockerfile (optional): Custom container definition - - requirements.txt (optional): Python dependencies - """ - - def __init__(self, workflows_dir: Path): - """ - Initialize workflow discovery. - - Args: - workflows_dir: Path to the workflows directory - """ - self.workflows_dir = workflows_dir - if not self.workflows_dir.exists(): - self.workflows_dir.mkdir(parents=True, exist_ok=True) - logger.info(f"Created workflows directory: {self.workflows_dir}") - - # Import registry - this validates it on import - try: - from toolbox.workflows.registry import WORKFLOW_REGISTRY, list_registered_workflows - self.registry = WORKFLOW_REGISTRY - logger.info(f"Loaded workflow registry with {len(self.registry)} registered workflows") - except ImportError as e: - logger.error(f"Failed to import workflow registry: {e}") - self.registry = {} - except Exception as e: - logger.error(f"Registry validation failed: {e}") - self.registry = {} - - # Cache for discovered workflows - self._workflow_cache: Optional[Dict[str, WorkflowInfo]] = None - self._cache_timestamp: Optional[float] = None - self._cache_ttl = 60.0 # Cache TTL in seconds - - async def discover_workflows(self) -> Dict[str, WorkflowInfo]: - """ - Discover workflows by cross-referencing filesystem with registry. - Uses caching to avoid frequent filesystem scans. - - Returns: - Dictionary mapping workflow names to their information - """ - # Check cache validity - import time - current_time = time.time() - - if (self._workflow_cache is not None and - self._cache_timestamp is not None and - (current_time - self._cache_timestamp) < self._cache_ttl): - # Return cached results - logger.debug(f"Returning cached workflow discovery ({len(self._workflow_cache)} workflows)") - return self._workflow_cache - workflows = {} - discovered_dirs = set() - registry_names = set(self.registry.keys()) - - if not self.workflows_dir.exists(): - logger.warning(f"Workflows directory does not exist: {self.workflows_dir}") - return workflows - - # Recursively scan all directories and subdirectories - await self._scan_directory_recursive(self.workflows_dir, workflows, discovered_dirs) - - # Check for registry entries without corresponding directories - missing_dirs = registry_names - discovered_dirs - if missing_dirs: - logger.warning( - f"Registry contains workflows without filesystem directories: {missing_dirs}. " - f"These workflows cannot be deployed." - ) - - logger.info( - f"Discovery complete: {len(workflows)} workflows ready for deployment, " - f"{len(missing_dirs)} registry entries missing directories, " - f"{len(discovered_dirs - registry_names)} filesystem workflows not registered" - ) - - # Update cache - self._workflow_cache = workflows - self._cache_timestamp = current_time - - return workflows - - async def _scan_directory_recursive(self, directory: Path, workflows: Dict[str, WorkflowInfo], discovered_dirs: set): - """ - Recursively scan directory for workflows. - - Args: - directory: Directory to scan - workflows: Dictionary to populate with discovered workflows - discovered_dirs: Set to track discovered workflow names - """ - for item in directory.iterdir(): - if not item.is_dir(): - continue - - if item.name.startswith('_') or item.name.startswith('.'): - continue # Skip hidden or private directories - - # Check if this directory contains workflow files (workflow.py and metadata.yaml) - workflow_file = item / "workflow.py" - metadata_file = item / "metadata.yaml" - - if workflow_file.exists() and metadata_file.exists(): - # This is a workflow directory - workflow_name = item.name - discovered_dirs.add(workflow_name) - - # Only process workflows that are in the registry - if workflow_name not in self.registry: - logger.warning( - f"Workflow '{workflow_name}' found in filesystem but not in registry. " - f"Add it to toolbox/workflows/registry.py to enable deployment." - ) - continue - - try: - workflow_info = await self._load_workflow(item) - if workflow_info: - workflows[workflow_info.name] = workflow_info - logger.info(f"Discovered and registered workflow: {workflow_info.name}") - except Exception as e: - logger.error(f"Failed to load workflow from {item}: {e}") - else: - # This is a category directory, recurse into it - await self._scan_directory_recursive(item, workflows, discovered_dirs) - - async def _load_workflow(self, workflow_dir: Path) -> Optional[WorkflowInfo]: - """ - Load and validate a single workflow. - - Args: - workflow_dir: Path to the workflow directory - - Returns: - WorkflowInfo if valid, None otherwise - """ - workflow_name = workflow_dir.name - - # Check for mandatory files - workflow_file = workflow_dir / "workflow.py" - metadata_file = workflow_dir / "metadata.yaml" - - if not workflow_file.exists(): - logger.warning(f"Workflow {workflow_name} missing workflow.py") - return None - - if not metadata_file.exists(): - logger.error(f"Workflow {workflow_name} missing mandatory metadata.yaml") - return None - - # Load and validate metadata - try: - metadata = self._load_metadata(metadata_file) - if not self._validate_metadata(metadata, workflow_name): - return None - except Exception as e: - logger.error(f"Failed to load metadata for {workflow_name}: {e}") - return None - - # Check for mandatory Dockerfile - dockerfile = workflow_dir / "Dockerfile" - if not dockerfile.exists(): - logger.error(f"Workflow {workflow_name} missing mandatory Dockerfile") - return None - - has_docker = True # Always True since Dockerfile is mandatory - - # Get flow function name from metadata or use default - flow_function_name = metadata.get("flow_function", "main_flow") - - return WorkflowInfo( - name=workflow_name, - path=workflow_dir, - workflow_file=workflow_file, - dockerfile=dockerfile, - has_docker=has_docker, - metadata=metadata, - flow_function_name=flow_function_name - ) - - def _load_metadata(self, metadata_file: Path) -> Dict[str, Any]: - """ - Load metadata from YAML file. - - Args: - metadata_file: Path to metadata.yaml - - Returns: - Dictionary containing metadata - """ - with open(metadata_file, 'r') as f: - metadata = yaml.safe_load(f) - - if metadata is None: - raise ValueError("Empty metadata file") - - return metadata - - def _validate_metadata(self, metadata: Dict[str, Any], workflow_name: str) -> bool: - """ - Validate that metadata contains all required fields. - - Args: - metadata: Metadata dictionary - workflow_name: Name of the workflow for logging - - Returns: - True if valid, False otherwise - """ - required_fields = ["name", "version", "description", "author", "category", "parameters", "requirements"] - - missing_fields = [] - for field in required_fields: - if field not in metadata: - missing_fields.append(field) - - if missing_fields: - logger.error( - f"Workflow {workflow_name} metadata missing required fields: {missing_fields}" - ) - return False - - # Validate version format (semantic versioning) - version = metadata.get("version", "") - if not self._is_valid_version(version): - logger.error(f"Workflow {workflow_name} has invalid version format: {version}") - return False - - # Validate parameters structure - parameters = metadata.get("parameters", {}) - if not isinstance(parameters, dict): - logger.error(f"Workflow {workflow_name} parameters must be a dictionary") - return False - - return True - - def _is_valid_version(self, version: str) -> bool: - """ - Check if version follows semantic versioning (x.y.z). - - Args: - version: Version string - - Returns: - True if valid semantic version - """ - try: - parts = version.split('.') - if len(parts) != 3: - return False - for part in parts: - int(part) # Check if each part is a number - return True - except (ValueError, AttributeError): - return False - - def invalidate_cache(self) -> None: - """ - Invalidate the workflow discovery cache. - Useful when workflows are added or modified. - """ - self._workflow_cache = None - self._cache_timestamp = None - logger.debug("Workflow discovery cache invalidated") - - def get_flow_function(self, workflow_name: str) -> Optional[Callable]: - """ - Get the flow function from the registry. - - Args: - workflow_name: Name of the workflow - - Returns: - The flow function if found in registry, None otherwise - """ - if workflow_name not in self.registry: - logger.error( - f"Workflow '{workflow_name}' not found in registry. " - f"Available workflows: {list(self.registry.keys())}" - ) - return None - - try: - from toolbox.workflows.registry import get_workflow_flow - flow_func = get_workflow_flow(workflow_name) - logger.debug(f"Retrieved flow function for '{workflow_name}' from registry") - return flow_func - except Exception as e: - logger.error(f"Failed to get flow function for '{workflow_name}': {e}") - return None - - def get_registry_info(self, workflow_name: str) -> Optional[Dict[str, Any]]: - """ - Get registry information for a workflow. - - Args: - workflow_name: Name of the workflow - - Returns: - Registry information if found, None otherwise - """ - if workflow_name not in self.registry: - return None - - try: - from toolbox.workflows.registry import get_workflow_info - return get_workflow_info(workflow_name) - except Exception as e: - logger.error(f"Failed to get registry info for '{workflow_name}': {e}") - return None - - @staticmethod - def get_metadata_schema() -> Dict[str, Any]: - """ - Get the JSON schema for workflow metadata. - - Returns: - JSON schema dictionary - """ - return { - "type": "object", - "required": ["name", "version", "description", "author", "category", "parameters", "requirements"], - "properties": { - "name": { - "type": "string", - "description": "Workflow name" - }, - "version": { - "type": "string", - "pattern": "^\\d+\\.\\d+\\.\\d+$", - "description": "Semantic version (x.y.z)" - }, - "description": { - "type": "string", - "description": "Workflow description" - }, - "author": { - "type": "string", - "description": "Workflow author" - }, - "category": { - "type": "string", - "enum": ["comprehensive", "specialized", "fuzzing", "focused"], - "description": "Workflow category" - }, - "tags": { - "type": "array", - "items": {"type": "string"}, - "description": "Workflow tags for categorization" - }, - "requirements": { - "type": "object", - "required": ["tools", "resources"], - "properties": { - "tools": { - "type": "array", - "items": {"type": "string"}, - "description": "Required security tools" - }, - "resources": { - "type": "object", - "required": ["memory", "cpu", "timeout"], - "properties": { - "memory": { - "type": "string", - "pattern": "^\\d+[GMK]i$", - "description": "Memory limit (e.g., 1Gi, 512Mi)" - }, - "cpu": { - "type": "string", - "pattern": "^\\d+m?$", - "description": "CPU limit (e.g., 1000m, 2)" - }, - "timeout": { - "type": "integer", - "minimum": 60, - "maximum": 7200, - "description": "Workflow timeout in seconds" - } - } - } - } - }, - "parameters": { - "type": "object", - "description": "Workflow parameters schema" - }, - "default_parameters": { - "type": "object", - "description": "Default parameter values" - }, - "required_modules": { - "type": "array", - "items": {"type": "string"}, - "description": "Required module names" - }, - "supported_volume_modes": { - "type": "array", - "items": {"enum": ["ro", "rw"]}, - "default": ["ro", "rw"], - "description": "Supported volume mount modes" - }, - "flow_function": { - "type": "string", - "default": "main_flow", - "description": "Name of the flow function in workflow.py" - } - } - } \ No newline at end of file diff --git a/backend/src/main.py b/backend/src/main.py index 6843a51..9866c43 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -12,7 +12,6 @@ import asyncio import logging import os -from uuid import UUID from contextlib import AsyncExitStack, asynccontextmanager, suppress from typing import Any, Dict, Optional, List @@ -23,31 +22,20 @@ from starlette.routing import Mount from fastmcp.server.http import create_sse_app -from src.core.prefect_manager import PrefectManager -from src.core.setup import setup_docker_pool, setup_result_storage, validate_infrastructure -from src.core.workflow_discovery import WorkflowDiscovery +from src.temporal.manager import TemporalManager +from src.core.setup import setup_result_storage, validate_infrastructure from src.api import workflows, runs, fuzzing -from src.services.prefect_stats_monitor import prefect_stats_monitor from fastmcp import FastMCP -from prefect.client.orchestration import get_client -from prefect.client.schemas.filters import ( - FlowRunFilter, - FlowRunFilterDeploymentId, - FlowRunFilterState, - FlowRunFilterStateType, -) -from prefect.client.schemas.sorting import FlowRunSort -from prefect.states import StateType logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -prefect_mgr = PrefectManager() +temporal_mgr = TemporalManager() -class PrefectBootstrapState: - """Tracks Prefect initialization progress for API and MCP consumers.""" +class TemporalBootstrapState: + """Tracks Temporal initialization progress for API and MCP consumers.""" def __init__(self) -> None: self.ready: bool = False @@ -64,19 +52,19 @@ class PrefectBootstrapState: } -prefect_bootstrap_state = PrefectBootstrapState() +temporal_bootstrap_state = TemporalBootstrapState() -# Configure retry strategy for bootstrapping Prefect + infrastructure +# Configure retry strategy for bootstrapping Temporal + infrastructure STARTUP_RETRY_SECONDS = max(1, int(os.getenv("FUZZFORGE_STARTUP_RETRY_SECONDS", "5"))) STARTUP_RETRY_MAX_SECONDS = max( STARTUP_RETRY_SECONDS, int(os.getenv("FUZZFORGE_STARTUP_RETRY_MAX_SECONDS", "60")), ) -prefect_bootstrap_task: Optional[asyncio.Task] = None +temporal_bootstrap_task: Optional[asyncio.Task] = None # --------------------------------------------------------------------------- -# FastAPI application (REST API remains unchanged) +# FastAPI application (REST API) # --------------------------------------------------------------------------- app = FastAPI( @@ -90,20 +78,19 @@ app.include_router(runs.router) app.include_router(fuzzing.router) -def get_prefect_status() -> Dict[str, Any]: - """Return a snapshot of Prefect bootstrap state for diagnostics.""" - status = prefect_bootstrap_state.as_dict() - status["workflows_loaded"] = len(prefect_mgr.workflows) - status["deployments_tracked"] = len(prefect_mgr.deployments) +def get_temporal_status() -> Dict[str, Any]: + """Return a snapshot of Temporal bootstrap state for diagnostics.""" + status = temporal_bootstrap_state.as_dict() + status["workflows_loaded"] = len(temporal_mgr.workflows) status["bootstrap_task_running"] = ( - prefect_bootstrap_task is not None and not prefect_bootstrap_task.done() + temporal_bootstrap_task is not None and not temporal_bootstrap_task.done() ) return status -def _prefect_not_ready_status() -> Optional[Dict[str, Any]]: - """Return status details if Prefect is not ready yet.""" - status = get_prefect_status() +def _temporal_not_ready_status() -> Optional[Dict[str, Any]]: + """Return status details if Temporal is not ready yet.""" + status = get_temporal_status() if status.get("ready"): return None return status @@ -111,19 +98,19 @@ def _prefect_not_ready_status() -> Optional[Dict[str, Any]]: @app.get("/") async def root() -> Dict[str, Any]: - status = get_prefect_status() + status = get_temporal_status() return { "name": "FuzzForge API", "version": "0.6.0", "status": "ready" if status.get("ready") else "initializing", "workflows_loaded": status.get("workflows_loaded", 0), - "prefect": status, + "temporal": status, } @app.get("/health") async def health() -> Dict[str, str]: - status = get_prefect_status() + status = get_temporal_status() health_status = "healthy" if status.get("ready") else "initializing" return {"status": health_status} @@ -165,65 +152,61 @@ _fastapi_mcp_imported = False mcp = FastMCP(name="FuzzForge MCP") -async def _bootstrap_prefect_with_retries() -> None: - """Initialize Prefect infrastructure with exponential backoff retries.""" +async def _bootstrap_temporal_with_retries() -> None: + """Initialize Temporal infrastructure with exponential backoff retries.""" attempt = 0 while True: attempt += 1 - prefect_bootstrap_state.task_running = True - prefect_bootstrap_state.status = "starting" - prefect_bootstrap_state.ready = False - prefect_bootstrap_state.last_error = None + temporal_bootstrap_state.task_running = True + temporal_bootstrap_state.status = "starting" + temporal_bootstrap_state.ready = False + temporal_bootstrap_state.last_error = None try: - logger.info("Bootstrapping Prefect infrastructure...") + logger.info("Bootstrapping Temporal infrastructure...") await validate_infrastructure() - await setup_docker_pool() await setup_result_storage() - await prefect_mgr.initialize() - await prefect_stats_monitor.start_monitoring() + await temporal_mgr.initialize() - prefect_bootstrap_state.ready = True - prefect_bootstrap_state.status = "ready" - prefect_bootstrap_state.task_running = False - logger.info("Prefect infrastructure ready") + temporal_bootstrap_state.ready = True + temporal_bootstrap_state.status = "ready" + temporal_bootstrap_state.task_running = False + logger.info("Temporal infrastructure ready") return except asyncio.CancelledError: - prefect_bootstrap_state.status = "cancelled" - prefect_bootstrap_state.task_running = False - logger.info("Prefect bootstrap task cancelled") + temporal_bootstrap_state.status = "cancelled" + temporal_bootstrap_state.task_running = False + logger.info("Temporal bootstrap task cancelled") raise except Exception as exc: # pragma: no cover - defensive logging on infra startup - logger.exception("Prefect bootstrap failed") - prefect_bootstrap_state.ready = False - prefect_bootstrap_state.status = "error" - prefect_bootstrap_state.last_error = str(exc) + logger.exception("Temporal bootstrap failed") + temporal_bootstrap_state.ready = False + temporal_bootstrap_state.status = "error" + temporal_bootstrap_state.last_error = str(exc) # Ensure partial initialization does not leave stale state behind - prefect_mgr.workflows.clear() - prefect_mgr.deployments.clear() - await prefect_stats_monitor.stop_monitoring() + temporal_mgr.workflows.clear() wait_time = min( STARTUP_RETRY_SECONDS * (2 ** (attempt - 1)), STARTUP_RETRY_MAX_SECONDS, ) - logger.info("Retrying Prefect bootstrap in %s second(s)", wait_time) + logger.info("Retrying Temporal bootstrap in %s second(s)", wait_time) try: await asyncio.sleep(wait_time) except asyncio.CancelledError: - prefect_bootstrap_state.status = "cancelled" - prefect_bootstrap_state.task_running = False + temporal_bootstrap_state.status = "cancelled" + temporal_bootstrap_state.task_running = False raise def _lookup_workflow(workflow_name: str): - info = prefect_mgr.workflows.get(workflow_name) + info = temporal_mgr.workflows.get(workflow_name) if not info: return None metadata = info.metadata @@ -248,24 +231,23 @@ def _lookup_workflow(workflow_name: str): "required_modules": metadata.get("required_modules", []), "supported_volume_modes": supported_modes, "default_target_path": default_target_path, - "default_volume_mode": default_volume_mode, - "has_custom_docker": bool(info.has_docker), + "default_volume_mode": default_volume_mode } @mcp.tool async def list_workflows_mcp() -> Dict[str, Any]: """List all discovered workflows and their metadata summary.""" - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { "workflows": [], - "prefect": not_ready, - "message": "Prefect infrastructure is still initializing", + "temporal": not_ready, + "message": "Temporal infrastructure is still initializing", } workflows_summary = [] - for name, info in prefect_mgr.workflows.items(): + for name, info in temporal_mgr.workflows.items(): metadata = info.metadata defaults = metadata.get("default_parameters", {}) workflows_summary.append({ @@ -279,20 +261,19 @@ async def list_workflows_mcp() -> Dict[str, Any]: or defaults.get("volume_mode") or "ro", "default_target_path": metadata.get("default_target_path") - or defaults.get("target_path"), - "has_custom_docker": bool(info.has_docker), + or defaults.get("target_path") }) - return {"workflows": workflows_summary, "prefect": get_prefect_status()} + return {"workflows": workflows_summary, "temporal": get_temporal_status()} @mcp.tool async def get_workflow_metadata_mcp(workflow_name: str) -> Dict[str, Any]: """Fetch detailed metadata for a workflow.""" - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } data = _lookup_workflow(workflow_name) @@ -304,11 +285,11 @@ async def get_workflow_metadata_mcp(workflow_name: str) -> Dict[str, Any]: @mcp.tool async def get_workflow_parameters_mcp(workflow_name: str) -> Dict[str, Any]: """Return the parameter schema and defaults for a workflow.""" - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } data = _lookup_workflow(workflow_name) @@ -323,72 +304,41 @@ async def get_workflow_parameters_mcp(workflow_name: str) -> Dict[str, Any]: @mcp.tool async def get_workflow_metadata_schema_mcp() -> Dict[str, Any]: """Return the JSON schema describing workflow metadata files.""" + from src.temporal.discovery import WorkflowDiscovery return WorkflowDiscovery.get_metadata_schema() @mcp.tool async def submit_security_scan_mcp( workflow_name: str, - target_path: str | None = None, - volume_mode: str | None = None, + target_id: str, parameters: Dict[str, Any] | None = None, ) -> Dict[str, Any] | Dict[str, str]: - """Submit a Prefect workflow via MCP.""" + """Submit a Temporal workflow via MCP.""" try: - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } - workflow_info = prefect_mgr.workflows.get(workflow_name) + workflow_info = temporal_mgr.workflows.get(workflow_name) if not workflow_info: return {"error": f"Workflow '{workflow_name}' not found"} metadata = workflow_info.metadata or {} defaults = metadata.get("default_parameters", {}) - resolved_target_path = target_path or metadata.get("default_target_path") or defaults.get("target_path") - if not resolved_target_path: - return { - "error": ( - "target_path is required and no default_target_path is defined in metadata" - ), - "metadata": { - "workflow": workflow_name, - "default_target_path": metadata.get("default_target_path"), - }, - } - - requested_volume_mode = volume_mode or metadata.get("default_volume_mode") or defaults.get("volume_mode") - if not requested_volume_mode: - requested_volume_mode = "ro" - - normalised_volume_mode = ( - str(requested_volume_mode).strip().lower().replace("-", "_") - ) - if normalised_volume_mode in {"read_only", "readonly", "ro"}: - normalised_volume_mode = "ro" - elif normalised_volume_mode in {"read_write", "readwrite", "rw"}: - normalised_volume_mode = "rw" - else: - supported_modes = metadata.get("supported_volume_modes", ["ro", "rw"]) - if isinstance(supported_modes, list) and normalised_volume_mode in supported_modes: - pass - else: - normalised_volume_mode = "ro" - parameters = parameters or {} - cleaned_parameters: Dict[str, Any] = {**defaults, **parameters} - # Ensure *_config structures default to dicts so Prefect validation passes. + # Ensure *_config structures default to dicts for key, value in list(cleaned_parameters.items()): if isinstance(key, str) and key.endswith("_config") and value is None: cleaned_parameters[key] = {} - # Some workflows expect configuration dictionaries even when omitted. + # Some workflows expect configuration dictionaries even when omitted parameter_definitions = ( metadata.get("parameters", {}).get("properties", {}) if isinstance(metadata.get("parameters"), dict) @@ -403,20 +353,19 @@ async def submit_security_scan_mcp( elif cleaned_parameters[key] is None: cleaned_parameters[key] = {} - flow_run = await prefect_mgr.submit_workflow( + # Start workflow + handle = await temporal_mgr.run_workflow( workflow_name=workflow_name, - target_path=resolved_target_path, - volume_mode=normalised_volume_mode, - parameters=cleaned_parameters, + target_id=target_id, + workflow_params=cleaned_parameters, ) return { - "run_id": str(flow_run.id), - "status": flow_run.state.name if flow_run.state else "PENDING", + "run_id": handle.id, + "status": "RUNNING", "workflow": workflow_name, "message": f"Workflow '{workflow_name}' submitted successfully", - "target_path": resolved_target_path, - "volume_mode": normalised_volume_mode, + "target_id": target_id, "parameters": cleaned_parameters, "mcp_enabled": True, } @@ -427,43 +376,38 @@ async def submit_security_scan_mcp( @mcp.tool async def get_comprehensive_scan_summary(run_id: str) -> Dict[str, Any] | Dict[str, str]: - """Return a summary for the given flow run via MCP.""" + """Return a summary for the given workflow run via MCP.""" try: - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } - status = await prefect_mgr.get_flow_run_status(run_id) - findings = await prefect_mgr.get_flow_run_findings(run_id) - - workflow_name = "unknown" - deployment_id = status.get("workflow", "") - for name, deployment in prefect_mgr.deployments.items(): - if str(deployment) == str(deployment_id): - workflow_name = name - break + status = await temporal_mgr.get_workflow_status(run_id) + # Try to get result if completed total_findings = 0 severity_summary = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0} - if findings and "sarif" in findings: - sarif = findings["sarif"] - if isinstance(sarif, dict): - total_findings = sarif.get("total_findings", 0) + if status.get("status") == "COMPLETED": + try: + result = await temporal_mgr.get_workflow_result(run_id) + if isinstance(result, dict): + summary = result.get("summary", {}) + total_findings = summary.get("total_findings", 0) + except Exception as e: + logger.debug(f"Could not retrieve result for {run_id}: {e}") return { "run_id": run_id, - "workflow": workflow_name, + "workflow": "unknown", # Temporal doesn't track workflow name in status "status": status.get("status", "unknown"), - "is_completed": status.get("is_completed", False), + "is_completed": status.get("status") == "COMPLETED", "total_findings": total_findings, "severity_summary": severity_summary, - "scan_duration": status.get("updated_at", "") - if status.get("is_completed") - else "In progress", + "scan_duration": status.get("close_time", "In progress"), "recommendations": ( [ "Review high and critical severity findings first", @@ -482,32 +426,26 @@ async def get_comprehensive_scan_summary(run_id: str) -> Dict[str, Any] | Dict[s @mcp.tool async def get_run_status_mcp(run_id: str) -> Dict[str, Any]: - """Return current status information for a Prefect run.""" + """Return current status information for a Temporal run.""" try: - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } - status = await prefect_mgr.get_flow_run_status(run_id) - workflow_name = "unknown" - deployment_id = status.get("workflow", "") - for name, deployment in prefect_mgr.deployments.items(): - if str(deployment) == str(deployment_id): - workflow_name = name - break + status = await temporal_mgr.get_workflow_status(run_id) return { - "run_id": status["run_id"], - "workflow": workflow_name, + "run_id": run_id, + "workflow": "unknown", "status": status["status"], - "is_completed": status["is_completed"], - "is_failed": status["is_failed"], - "is_running": status["is_running"], - "created_at": status["created_at"], - "updated_at": status["updated_at"], + "is_completed": status["status"] in ["COMPLETED", "FAILED", "CANCELLED"], + "is_failed": status["status"] == "FAILED", + "is_running": status["status"] == "RUNNING", + "created_at": status.get("start_time"), + "updated_at": status.get("close_time") or status.get("execution_time"), } except Exception as exc: logger.exception("MCP run status failed") @@ -518,38 +456,30 @@ async def get_run_status_mcp(run_id: str) -> Dict[str, Any]: async def get_run_findings_mcp(run_id: str) -> Dict[str, Any]: """Return SARIF findings for a completed run.""" try: - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } - status = await prefect_mgr.get_flow_run_status(run_id) - if not status.get("is_completed"): + status = await temporal_mgr.get_workflow_status(run_id) + if status.get("status") != "COMPLETED": return {"error": f"Run {run_id} not completed. Status: {status.get('status')}"} - findings = await prefect_mgr.get_flow_run_findings(run_id) - - workflow_name = "unknown" - deployment_id = status.get("workflow", "") - for name, deployment in prefect_mgr.deployments.items(): - if str(deployment) == str(deployment_id): - workflow_name = name - break + result = await temporal_mgr.get_workflow_result(run_id) metadata = { - "completion_time": status.get("updated_at"), + "completion_time": status.get("close_time"), "workflow_version": "unknown", } - info = prefect_mgr.workflows.get(workflow_name) - if info: - metadata["workflow_version"] = info.metadata.get("version", "unknown") + + sarif = result.get("sarif", {}) if isinstance(result, dict) else {} return { - "workflow": workflow_name, + "workflow": "unknown", "run_id": run_id, - "sarif": findings, + "sarif": sarif, "metadata": metadata, } except Exception as exc: @@ -561,16 +491,15 @@ async def get_run_findings_mcp(run_id: str) -> Dict[str, Any]: async def list_recent_runs_mcp( limit: int = 10, workflow_name: str | None = None, - states: List[str] | None = None, ) -> Dict[str, Any]: - """List recent Prefect runs with optional workflow/state filters.""" + """List recent Temporal runs with optional workflow filter.""" - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { "runs": [], - "prefect": not_ready, - "message": "Prefect infrastructure is still initializing", + "temporal": not_ready, + "message": "Temporal infrastructure is still initializing", } try: @@ -579,116 +508,49 @@ async def list_recent_runs_mcp( limit_value = 10 limit_value = max(1, min(limit_value, 100)) - deployment_map = { - str(deployment_id): workflow - for workflow, deployment_id in prefect_mgr.deployments.items() - } + try: + # Build filter query + filter_query = None + if workflow_name: + workflow_info = temporal_mgr.workflows.get(workflow_name) + if workflow_info: + filter_query = f'WorkflowType="{workflow_info.workflow_type}"' - deployment_filter_value = None - if workflow_name: - deployment_id = prefect_mgr.deployments.get(workflow_name) - if not deployment_id: - return { - "runs": [], - "prefect": get_prefect_status(), - "error": f"Workflow '{workflow_name}' has no registered deployment", - } - try: - deployment_filter_value = UUID(str(deployment_id)) - except ValueError: - return { - "runs": [], - "prefect": get_prefect_status(), - "error": ( - f"Deployment id '{deployment_id}' for workflow '{workflow_name}' is invalid" - ), - } + workflows = await temporal_mgr.list_workflows(filter_query, limit_value) - desired_state_types: List[StateType] = [] - if states: - for raw_state in states: - if not raw_state: - continue - normalised = raw_state.strip().upper() - if normalised == "ALL": - desired_state_types = [] - break - try: - desired_state_types.append(StateType[normalised]) - except KeyError: - continue - if not desired_state_types: - desired_state_types = [ - StateType.RUNNING, - StateType.COMPLETED, - StateType.FAILED, - StateType.CANCELLED, - ] + results: List[Dict[str, Any]] = [] + for wf in workflows: + results.append({ + "run_id": wf["workflow_id"], + "workflow": workflow_name or "unknown", + "state": wf["status"], + "state_type": wf["status"], + "is_completed": wf["status"] in ["COMPLETED", "FAILED", "CANCELLED"], + "is_running": wf["status"] == "RUNNING", + "is_failed": wf["status"] == "FAILED", + "created_at": wf.get("start_time"), + "updated_at": wf.get("close_time"), + }) - flow_filter = FlowRunFilter() - if desired_state_types: - flow_filter.state = FlowRunFilterState( - type=FlowRunFilterStateType(any_=desired_state_types) - ) - if deployment_filter_value: - flow_filter.deployment_id = FlowRunFilterDeploymentId( - any_=[deployment_filter_value] - ) + return {"runs": results, "temporal": get_temporal_status()} - async with get_client() as client: - flow_runs = await client.read_flow_runs( - limit=limit_value, - flow_run_filter=flow_filter, - sort=FlowRunSort.START_TIME_DESC, - ) - - results: List[Dict[str, Any]] = [] - for flow_run in flow_runs: - deployment_id = getattr(flow_run, "deployment_id", None) - workflow = deployment_map.get(str(deployment_id), "unknown") - state = getattr(flow_run, "state", None) - state_name = getattr(state, "name", None) if state else None - state_type = getattr(state, "type", None) if state else None - - results.append( - { - "run_id": str(flow_run.id), - "workflow": workflow, - "deployment_id": str(deployment_id) if deployment_id else None, - "state": state_name or (state_type.name if state_type else None), - "state_type": state_type.name if state_type else None, - "is_completed": bool(getattr(state, "is_completed", lambda: False)()), - "is_running": bool(getattr(state, "is_running", lambda: False)()), - "is_failed": bool(getattr(state, "is_failed", lambda: False)()), - "created_at": getattr(flow_run, "created", None), - "updated_at": getattr(flow_run, "updated", None), - "expected_start_time": getattr(flow_run, "expected_start_time", None), - "start_time": getattr(flow_run, "start_time", None), - } - ) - - # Normalise datetimes to ISO 8601 strings for serialization - for entry in results: - for key in ("created_at", "updated_at", "expected_start_time", "start_time"): - value = entry.get(key) - if value is None: - continue - try: - entry[key] = value.isoformat() - except AttributeError: - entry[key] = str(value) - - return {"runs": results, "prefect": get_prefect_status()} + except Exception as exc: + logger.exception("Failed to list runs") + return { + "runs": [], + "temporal": get_temporal_status(), + "error": str(exc) + } @mcp.tool async def get_fuzzing_stats_mcp(run_id: str) -> Dict[str, Any]: """Return fuzzing statistics for a run if available.""" - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } stats = fuzzing.fuzzing_stats.get(run_id) @@ -708,11 +570,11 @@ async def get_fuzzing_stats_mcp(run_id: str) -> Dict[str, Any]: @mcp.tool async def get_fuzzing_crash_reports_mcp(run_id: str) -> Dict[str, Any]: """Return crash reports collected for a fuzzing run.""" - not_ready = _prefect_not_ready_status() + not_ready = _temporal_not_ready_status() if not_ready: return { - "error": "Prefect infrastructure not ready", - "prefect": not_ready, + "error": "Temporal infrastructure not ready", + "temporal": not_ready, } reports = fuzzing.crash_reports.get(run_id) @@ -725,11 +587,11 @@ async def get_fuzzing_crash_reports_mcp(run_id: str) -> Dict[str, Any]: async def get_backend_status_mcp() -> Dict[str, Any]: """Expose backend readiness, workflows, and registered MCP tools.""" - status = get_prefect_status() - response: Dict[str, Any] = {"prefect": status} + status = get_temporal_status() + response: Dict[str, Any] = {"temporal": status} if status.get("ready"): - response["workflows"] = list(prefect_mgr.workflows.keys()) + response["workflows"] = list(temporal_mgr.workflows.keys()) try: tools = await mcp._tool_manager.list_tools() @@ -775,12 +637,12 @@ def create_mcp_transport_app() -> Starlette: # --------------------------------------------------------------------------- -# Combined lifespan: Prefect init + dedicated MCP transports +# Combined lifespan: Temporal init + dedicated MCP transports # --------------------------------------------------------------------------- @asynccontextmanager async def combined_lifespan(app: FastAPI): - global prefect_bootstrap_task, _fastapi_mcp_imported + global temporal_bootstrap_task, _fastapi_mcp_imported logger.info("Starting FuzzForge backend...") @@ -793,12 +655,12 @@ async def combined_lifespan(app: FastAPI): except Exception as exc: logger.exception("Failed to import FastAPI endpoints into MCP", exc_info=exc) - # Kick off Prefect bootstrap in the background if needed - if prefect_bootstrap_task is None or prefect_bootstrap_task.done(): - prefect_bootstrap_task = asyncio.create_task(_bootstrap_prefect_with_retries()) - logger.info("Prefect bootstrap task started") + # Kick off Temporal bootstrap in the background if needed + if temporal_bootstrap_task is None or temporal_bootstrap_task.done(): + temporal_bootstrap_task = asyncio.create_task(_bootstrap_temporal_with_retries()) + logger.info("Temporal bootstrap task started") else: - logger.info("Prefect bootstrap task already running") + logger.info("Temporal bootstrap task already running") # Start MCP transports on shared port (HTTP + SSE) mcp_app = create_mcp_transport_app() @@ -846,18 +708,17 @@ async def combined_lifespan(app: FastAPI): mcp_server.force_exit = True await asyncio.gather(mcp_task, return_exceptions=True) - if prefect_bootstrap_task and not prefect_bootstrap_task.done(): - prefect_bootstrap_task.cancel() + if temporal_bootstrap_task and not temporal_bootstrap_task.done(): + temporal_bootstrap_task.cancel() with suppress(asyncio.CancelledError): - await prefect_bootstrap_task - prefect_bootstrap_state.task_running = False - if not prefect_bootstrap_state.ready: - prefect_bootstrap_state.status = "stopped" - prefect_bootstrap_state.next_retry_seconds = None - prefect_bootstrap_task = None + await temporal_bootstrap_task + temporal_bootstrap_state.task_running = False + if not temporal_bootstrap_state.ready: + temporal_bootstrap_state.status = "stopped" + temporal_bootstrap_task = None - logger.info("Shutting down Prefect statistics monitor...") - await prefect_stats_monitor.stop_monitoring() + # Close Temporal client + await temporal_mgr.close() logger.info("Shutting down FuzzForge backend...") diff --git a/backend/src/models/findings.py b/backend/src/models/findings.py index 05385d9..ddc756a 100644 --- a/backend/src/models/findings.py +++ b/backend/src/models/findings.py @@ -13,10 +13,9 @@ Models for workflow findings and submissions # # Additional attribution and requirements are provided in the NOTICE file. -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field from typing import Dict, Any, Optional, Literal, List from datetime import datetime -from pathlib import Path class WorkflowFindings(BaseModel): @@ -27,47 +26,13 @@ class WorkflowFindings(BaseModel): metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata") -class ResourceLimits(BaseModel): - """Resource limits for workflow execution""" - cpu_limit: Optional[str] = Field(None, description="CPU limit (e.g., '2' for 2 cores, '500m' for 0.5 cores)") - memory_limit: Optional[str] = Field(None, description="Memory limit (e.g., '1Gi', '512Mi')") - cpu_request: Optional[str] = Field(None, description="CPU request (guaranteed)") - memory_request: Optional[str] = Field(None, description="Memory request (guaranteed)") - - -class VolumeMount(BaseModel): - """Volume mount specification""" - host_path: str = Field(..., description="Host path to mount") - container_path: str = Field(..., description="Container path for mount") - mode: Literal["ro", "rw"] = Field(default="ro", description="Mount mode") - - @field_validator("host_path") - @classmethod - def validate_host_path(cls, v): - """Validate that the host path is absolute (existence checked at runtime)""" - path = Path(v) - if not path.is_absolute(): - raise ValueError(f"Host path must be absolute: {v}") - # Note: Path existence is validated at workflow runtime - # We can't validate existence here as this runs inside Docker container - return str(path) - - @field_validator("container_path") - @classmethod - def validate_container_path(cls, v): - """Validate that the container path is absolute""" - if not v.startswith('/'): - raise ValueError(f"Container path must be absolute: {v}") - return v - - class WorkflowSubmission(BaseModel): - """Submit a workflow with configurable settings""" - target_path: str = Field(..., description="Absolute path to analyze") - volume_mode: Literal["ro", "rw"] = Field( - default="ro", - description="Volume mount mode: read-only (ro) or read-write (rw)" - ) + """ + Submit a workflow with configurable settings. + + Note: This model is deprecated in favor of the /upload-and-submit endpoint + which handles file uploads directly. + """ parameters: Dict[str, Any] = Field( default_factory=dict, description="Workflow-specific parameters" @@ -78,25 +43,6 @@ class WorkflowSubmission(BaseModel): ge=1, le=604800 # Max 7 days to support fuzzing campaigns ) - resource_limits: Optional[ResourceLimits] = Field( - None, - description="Resource limits for workflow container" - ) - additional_volumes: List[VolumeMount] = Field( - default_factory=list, - description="Additional volume mounts (e.g., for corpus, output directories)" - ) - - @field_validator("target_path") - @classmethod - def validate_path(cls, v): - """Validate that the target path is absolute (existence checked at runtime)""" - path = Path(v) - if not path.is_absolute(): - raise ValueError(f"Path must be absolute: {v}") - # Note: Path existence is validated at workflow runtime when volumes are mounted - # We can't validate existence here as this runs inside Docker container - return str(path) class WorkflowStatus(BaseModel): @@ -131,10 +77,6 @@ class WorkflowMetadata(BaseModel): default=["ro", "rw"], description="Supported volume mount modes" ) - has_custom_docker: bool = Field( - default=False, - description="Whether workflow has custom Dockerfile" - ) class WorkflowListItem(BaseModel): diff --git a/backend/src/services/prefect_stats_monitor.py b/backend/src/services/prefect_stats_monitor.py deleted file mode 100644 index a46d88a..0000000 --- a/backend/src/services/prefect_stats_monitor.py +++ /dev/null @@ -1,394 +0,0 @@ -""" -Generic Prefect Statistics Monitor Service - -This service monitors ALL workflows for structured live data logging and -updates the appropriate statistics APIs. Works with any workflow that follows -the standard LIVE_STATS logging pattern. -""" -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - - -import asyncio -import json -import logging -from datetime import datetime, timedelta, timezone -from typing import Dict, Any, Optional -from prefect.client.orchestration import get_client -from prefect.client.schemas.objects import FlowRun, TaskRun -from src.models.findings import FuzzingStats -from src.api.fuzzing import fuzzing_stats, initialize_fuzzing_tracking, active_connections - -logger = logging.getLogger(__name__) - - -class PrefectStatsMonitor: - """Monitors Prefect flows and tasks for live statistics from any workflow""" - - def __init__(self): - self.monitoring = False - self.monitor_task = None - self.monitored_runs = set() - self.last_log_ts: Dict[str, datetime] = {} - self._client = None - self._client_refresh_time = None - self._client_refresh_interval = 300 # Refresh connection every 5 minutes - - async def start_monitoring(self): - """Start the Prefect statistics monitoring service""" - if self.monitoring: - logger.warning("Prefect stats monitor already running") - return - - self.monitoring = True - self.monitor_task = asyncio.create_task(self._monitor_flows()) - logger.info("Started Prefect statistics monitor") - - async def stop_monitoring(self): - """Stop the monitoring service""" - self.monitoring = False - if self.monitor_task: - self.monitor_task.cancel() - try: - await self.monitor_task - except asyncio.CancelledError: - pass - logger.info("Stopped Prefect statistics monitor") - - async def _get_or_refresh_client(self): - """Get or refresh Prefect client with connection pooling.""" - now = datetime.now(timezone.utc) - - if (self._client is None or - self._client_refresh_time is None or - (now - self._client_refresh_time).total_seconds() > self._client_refresh_interval): - - if self._client: - try: - await self._client.aclose() - except Exception: - pass - - self._client = get_client() - self._client_refresh_time = now - await self._client.__aenter__() - - return self._client - - async def _monitor_flows(self): - """Main monitoring loop that watches Prefect flows""" - try: - while self.monitoring: - try: - # Use connection pooling for better performance - client = await self._get_or_refresh_client() - - # Get recent flow runs (limit to reduce load) - flow_runs = await client.read_flow_runs( - limit=50, - sort="START_TIME_DESC", - ) - - # Only consider runs from the last 15 minutes - recent_cutoff = datetime.now(timezone.utc) - timedelta(minutes=15) - for flow_run in flow_runs: - created = getattr(flow_run, "created", None) - if created is None: - continue - try: - # Ensure timezone-aware comparison - if created.tzinfo is None: - created = created.replace(tzinfo=timezone.utc) - if created >= recent_cutoff: - await self._monitor_flow_run(client, flow_run) - except Exception: - # If comparison fails, attempt monitoring anyway - await self._monitor_flow_run(client, flow_run) - - await asyncio.sleep(5) # Check every 5 seconds - - except Exception as e: - logger.error(f"Error in Prefect monitoring: {e}") - await asyncio.sleep(10) - - except asyncio.CancelledError: - logger.info("Prefect monitoring cancelled") - except Exception as e: - logger.error(f"Fatal error in Prefect monitoring: {e}") - finally: - # Clean up client on exit - if self._client: - try: - await self._client.__aexit__(None, None, None) - except Exception: - pass - self._client = None - - async def _monitor_flow_run(self, client, flow_run: FlowRun): - """Monitor a specific flow run for statistics""" - run_id = str(flow_run.id) - workflow_name = flow_run.name or "unknown" - - try: - # Initialize tracking if not exists - only for workflows that might have live stats - if run_id not in fuzzing_stats: - initialize_fuzzing_tracking(run_id, workflow_name) - self.monitored_runs.add(run_id) - - # Skip corrupted entries (should not happen after startup cleanup, but defensive) - elif not isinstance(fuzzing_stats[run_id], FuzzingStats): - logger.warning(f"Skipping corrupted stats entry for {run_id}, reinitializing") - initialize_fuzzing_tracking(run_id, workflow_name) - self.monitored_runs.add(run_id) - - # Get task runs for this flow - task_runs = await client.read_task_runs( - flow_run_filter={"id": {"any_": [flow_run.id]}}, - limit=25, - ) - - # Check all tasks for live statistics logging - for task_run in task_runs: - await self._extract_stats_from_task(client, run_id, task_run, workflow_name) - - # Also scan flow-level logs as a fallback - await self._extract_stats_from_flow_logs(client, run_id, flow_run, workflow_name) - - except Exception as e: - logger.warning(f"Error monitoring flow run {run_id}: {e}") - - async def _extract_stats_from_task(self, client, run_id: str, task_run: TaskRun, workflow_name: str): - """Extract statistics from any task that logs live stats""" - try: - # Get task run logs - logs = await client.read_logs( - log_filter={ - "task_run_id": {"any_": [task_run.id]} - }, - limit=100, - sort="TIMESTAMP_ASC" - ) - - # Parse logs for LIVE_STATS entries (generic pattern for any workflow) - latest_stats = None - for log in logs: - # Prefer structured extra field if present - extra_data = getattr(log, "extra", None) or getattr(log, "extra_fields", None) or None - if isinstance(extra_data, dict): - stat_type = extra_data.get("stats_type") - if stat_type in ["fuzzing_live_update", "scan_progress", "analysis_update", "live_stats"]: - latest_stats = extra_data - continue - - # Fallback to parsing from message text - if ("FUZZ_STATS" in log.message or "LIVE_STATS" in log.message): - stats = self._parse_stats_from_log(log.message) - if stats: - latest_stats = stats - - # Update statistics if we found any - if latest_stats: - # Calculate elapsed time from task start - elapsed_time = 0 - if task_run.start_time: - # Ensure timezone-aware arithmetic - now = datetime.now(timezone.utc) - try: - elapsed_time = int((now - task_run.start_time).total_seconds()) - except Exception: - # Fallback to naive UTC if types mismatch - elapsed_time = int((datetime.utcnow() - task_run.start_time.replace(tzinfo=None)).total_seconds()) - - updated_stats = FuzzingStats( - run_id=run_id, - workflow=workflow_name, - executions=latest_stats.get("executions", 0), - executions_per_sec=latest_stats.get("executions_per_sec", 0.0), - crashes=latest_stats.get("crashes", 0), - unique_crashes=latest_stats.get("unique_crashes", 0), - corpus_size=latest_stats.get("corpus_size", 0), - elapsed_time=elapsed_time - ) - - # Update the global stats - previous = fuzzing_stats.get(run_id) - fuzzing_stats[run_id] = updated_stats - - # Broadcast to any active WebSocket clients for this run - if active_connections.get(run_id): - # Handle both Pydantic objects and plain dicts - if isinstance(updated_stats, dict): - stats_data = updated_stats - elif hasattr(updated_stats, 'model_dump'): - stats_data = updated_stats.model_dump() - elif hasattr(updated_stats, 'dict'): - stats_data = updated_stats.dict() - else: - stats_data = updated_stats.__dict__ - - message = { - "type": "stats_update", - "data": stats_data, - } - disconnected = [] - for ws in active_connections[run_id]: - try: - await ws.send_text(json.dumps(message)) - except Exception: - disconnected.append(ws) - # Clean up disconnected sockets - for ws in disconnected: - try: - active_connections[run_id].remove(ws) - except ValueError: - pass - - logger.debug(f"Updated Prefect stats for {run_id}: {updated_stats.executions} execs") - - except Exception as e: - logger.warning(f"Error extracting stats from task {task_run.id}: {e}") - - async def _extract_stats_from_flow_logs(self, client, run_id: str, flow_run: FlowRun, workflow_name: str): - """Extract statistics by scanning flow-level logs for LIVE/FUZZ stats""" - try: - logs = await client.read_logs( - log_filter={ - "flow_run_id": {"any_": [flow_run.id]} - }, - limit=200, - sort="TIMESTAMP_ASC" - ) - - latest_stats = None - last_seen = self.last_log_ts.get(run_id) - max_ts = last_seen - - for log in logs: - # Skip logs we've already processed - ts = getattr(log, "timestamp", None) - if last_seen and ts and ts <= last_seen: - continue - if ts and (max_ts is None or ts > max_ts): - max_ts = ts - - # Prefer structured extra field if available - extra_data = getattr(log, "extra", None) or getattr(log, "extra_fields", None) or None - if isinstance(extra_data, dict): - stat_type = extra_data.get("stats_type") - if stat_type in ["fuzzing_live_update", "scan_progress", "analysis_update", "live_stats"]: - latest_stats = extra_data - continue - - # Fallback to message parse - if ("FUZZ_STATS" in log.message or "LIVE_STATS" in log.message): - stats = self._parse_stats_from_log(log.message) - if stats: - latest_stats = stats - - if max_ts: - self.last_log_ts[run_id] = max_ts - - if latest_stats: - # Use flow_run timestamps for elapsed time if available - elapsed_time = 0 - start_time = getattr(flow_run, "start_time", None) or getattr(flow_run, "start_time", None) - if start_time: - now = datetime.now(timezone.utc) - try: - if start_time.tzinfo is None: - start_time = start_time.replace(tzinfo=timezone.utc) - elapsed_time = int((now - start_time).total_seconds()) - except Exception: - elapsed_time = int((datetime.utcnow() - start_time.replace(tzinfo=None)).total_seconds()) - - updated_stats = FuzzingStats( - run_id=run_id, - workflow=workflow_name, - executions=latest_stats.get("executions", 0), - executions_per_sec=latest_stats.get("executions_per_sec", 0.0), - crashes=latest_stats.get("crashes", 0), - unique_crashes=latest_stats.get("unique_crashes", 0), - corpus_size=latest_stats.get("corpus_size", 0), - elapsed_time=elapsed_time - ) - - fuzzing_stats[run_id] = updated_stats - - # Broadcast if listeners exist - if active_connections.get(run_id): - # Handle both Pydantic objects and plain dicts - if isinstance(updated_stats, dict): - stats_data = updated_stats - elif hasattr(updated_stats, 'model_dump'): - stats_data = updated_stats.model_dump() - elif hasattr(updated_stats, 'dict'): - stats_data = updated_stats.dict() - else: - stats_data = updated_stats.__dict__ - - message = { - "type": "stats_update", - "data": stats_data, - } - disconnected = [] - for ws in active_connections[run_id]: - try: - await ws.send_text(json.dumps(message)) - except Exception: - disconnected.append(ws) - for ws in disconnected: - try: - active_connections[run_id].remove(ws) - except ValueError: - pass - - except Exception as e: - logger.warning(f"Error extracting stats from flow logs {run_id}: {e}") - - def _parse_stats_from_log(self, log_message: str) -> Optional[Dict[str, Any]]: - """Parse statistics from a log message""" - try: - import re - - # Prefer explicit JSON after marker tokens - m = re.search(r'(?:FUZZ_STATS|LIVE_STATS)\s+(\{.*\})', log_message) - if m: - try: - return json.loads(m.group(1)) - except Exception: - pass - - # Fallback: Extract the extra= dict and coerce to JSON - stats_match = re.search(r'extra=({.*?})', log_message) - if not stats_match: - return None - - extra_str = stats_match.group(1) - extra_str = extra_str.replace("'", '"') - extra_str = extra_str.replace('None', 'null') - extra_str = extra_str.replace('True', 'true') - extra_str = extra_str.replace('False', 'false') - - stats_data = json.loads(extra_str) - - # Support multiple stat types for different workflows - stat_type = stats_data.get("stats_type") - if stat_type in ["fuzzing_live_update", "scan_progress", "analysis_update", "live_stats"]: - return stats_data - - except Exception as e: - logger.debug(f"Error parsing log stats: {e}") - - return None - - -# Global instance -prefect_stats_monitor = PrefectStatsMonitor() diff --git a/backend/src/storage/__init__.py b/backend/src/storage/__init__.py new file mode 100644 index 0000000..4f78cff --- /dev/null +++ b/backend/src/storage/__init__.py @@ -0,0 +1,10 @@ +""" +Storage abstraction layer for FuzzForge. + +Provides unified interface for storing and retrieving targets and results. +""" + +from .base import StorageBackend +from .s3_cached import S3CachedStorage + +__all__ = ["StorageBackend", "S3CachedStorage"] diff --git a/backend/src/storage/base.py b/backend/src/storage/base.py new file mode 100644 index 0000000..7323fd3 --- /dev/null +++ b/backend/src/storage/base.py @@ -0,0 +1,153 @@ +""" +Base storage backend interface. + +All storage implementations must implement this interface. +""" + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional, Dict, Any + + +class StorageBackend(ABC): + """ + Abstract base class for storage backends. + + Implementations handle storage and retrieval of: + - Uploaded targets (code, binaries, etc.) + - Workflow results + - Temporary files + """ + + @abstractmethod + async def upload_target( + self, + file_path: Path, + user_id: str, + metadata: Optional[Dict[str, Any]] = None + ) -> str: + """ + Upload a target file to storage. + + Args: + file_path: Local path to file to upload + user_id: ID of user uploading the file + metadata: Optional metadata to store with file + + Returns: + Target ID (unique identifier for retrieval) + + Raises: + FileNotFoundError: If file_path doesn't exist + StorageError: If upload fails + """ + pass + + @abstractmethod + async def get_target(self, target_id: str) -> Path: + """ + Get target file from storage. + + Args: + target_id: Unique identifier from upload_target() + + Returns: + Local path to cached file + + Raises: + FileNotFoundError: If target doesn't exist + StorageError: If download fails + """ + pass + + @abstractmethod + async def delete_target(self, target_id: str) -> None: + """ + Delete target from storage. + + Args: + target_id: Unique identifier to delete + + Raises: + StorageError: If deletion fails (doesn't raise if not found) + """ + pass + + @abstractmethod + async def upload_results( + self, + workflow_id: str, + results: Dict[str, Any], + results_format: str = "json" + ) -> str: + """ + Upload workflow results to storage. + + Args: + workflow_id: Workflow execution ID + results: Results dictionary + results_format: Format (json, sarif, etc.) + + Returns: + URL to uploaded results + + Raises: + StorageError: If upload fails + """ + pass + + @abstractmethod + async def get_results(self, workflow_id: str) -> Dict[str, Any]: + """ + Get workflow results from storage. + + Args: + workflow_id: Workflow execution ID + + Returns: + Results dictionary + + Raises: + FileNotFoundError: If results don't exist + StorageError: If download fails + """ + pass + + @abstractmethod + async def list_targets( + self, + user_id: Optional[str] = None, + limit: int = 100 + ) -> list[Dict[str, Any]]: + """ + List uploaded targets. + + Args: + user_id: Filter by user ID (None = all users) + limit: Maximum number of results + + Returns: + List of target metadata dictionaries + + Raises: + StorageError: If listing fails + """ + pass + + @abstractmethod + async def cleanup_cache(self) -> int: + """ + Clean up local cache (LRU eviction). + + Returns: + Number of files removed + + Raises: + StorageError: If cleanup fails + """ + pass + + +class StorageError(Exception): + """Base exception for storage operations.""" + pass diff --git a/backend/src/storage/s3_cached.py b/backend/src/storage/s3_cached.py new file mode 100644 index 0000000..99c8e3a --- /dev/null +++ b/backend/src/storage/s3_cached.py @@ -0,0 +1,423 @@ +""" +S3-compatible storage backend with local caching. + +Works with MinIO (dev/prod) or AWS S3 (cloud). +""" + +import json +import logging +import os +import shutil +from datetime import datetime +from pathlib import Path +from typing import Optional, Dict, Any +from uuid import uuid4 + +import boto3 +from botocore.exceptions import ClientError + +from .base import StorageBackend, StorageError + +logger = logging.getLogger(__name__) + + +class S3CachedStorage(StorageBackend): + """ + S3-compatible storage with local caching. + + Features: + - Upload targets to S3/MinIO + - Download with local caching (LRU eviction) + - Lifecycle management (auto-cleanup old files) + - Metadata tracking + """ + + def __init__( + self, + endpoint_url: Optional[str] = None, + access_key: Optional[str] = None, + secret_key: Optional[str] = None, + bucket: str = "targets", + region: str = "us-east-1", + use_ssl: bool = False, + cache_dir: Optional[Path] = None, + cache_max_size_gb: int = 10 + ): + """ + Initialize S3 storage backend. + + Args: + endpoint_url: S3 endpoint (None = AWS S3, or MinIO URL) + access_key: S3 access key (None = from env) + secret_key: S3 secret key (None = from env) + bucket: S3 bucket name + region: AWS region + use_ssl: Use HTTPS + cache_dir: Local cache directory + cache_max_size_gb: Maximum cache size in GB + """ + # Use environment variables as defaults + self.endpoint_url = endpoint_url or os.getenv('S3_ENDPOINT', 'http://minio:9000') + self.access_key = access_key or os.getenv('S3_ACCESS_KEY', 'fuzzforge') + self.secret_key = secret_key or os.getenv('S3_SECRET_KEY', 'fuzzforge123') + self.bucket = bucket or os.getenv('S3_BUCKET', 'targets') + self.region = region or os.getenv('S3_REGION', 'us-east-1') + self.use_ssl = use_ssl or os.getenv('S3_USE_SSL', 'false').lower() == 'true' + + # Cache configuration + self.cache_dir = cache_dir or Path(os.getenv('CACHE_DIR', '/tmp/fuzzforge-cache')) + self.cache_max_size = cache_max_size_gb * (1024 ** 3) # Convert to bytes + + # Ensure cache directory exists + self.cache_dir.mkdir(parents=True, exist_ok=True) + + # Initialize S3 client + try: + self.s3_client = boto3.client( + 's3', + endpoint_url=self.endpoint_url, + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + region_name=self.region, + use_ssl=self.use_ssl + ) + logger.info(f"Initialized S3 storage: {self.endpoint_url}/{self.bucket}") + except Exception as e: + logger.error(f"Failed to initialize S3 client: {e}") + raise StorageError(f"S3 initialization failed: {e}") + + async def upload_target( + self, + file_path: Path, + user_id: str, + metadata: Optional[Dict[str, Any]] = None + ) -> str: + """Upload target file to S3/MinIO.""" + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + # Generate unique target ID + target_id = str(uuid4()) + + # Prepare metadata + upload_metadata = { + 'user_id': user_id, + 'uploaded_at': datetime.now().isoformat(), + 'filename': file_path.name, + 'size': str(file_path.stat().st_size) + } + if metadata: + upload_metadata.update(metadata) + + # Upload to S3 + s3_key = f'{target_id}/target' + try: + logger.info(f"Uploading target to s3://{self.bucket}/{s3_key}") + + self.s3_client.upload_file( + str(file_path), + self.bucket, + s3_key, + ExtraArgs={ + 'Metadata': upload_metadata + } + ) + + file_size_mb = file_path.stat().st_size / (1024 * 1024) + logger.info( + f"āœ“ Uploaded target {target_id} " + f"({file_path.name}, {file_size_mb:.2f} MB)" + ) + + return target_id + + except ClientError as e: + logger.error(f"S3 upload failed: {e}", exc_info=True) + raise StorageError(f"Failed to upload target: {e}") + except Exception as e: + logger.error(f"Upload failed: {e}", exc_info=True) + raise StorageError(f"Upload error: {e}") + + async def get_target(self, target_id: str) -> Path: + """Get target from cache or download from S3/MinIO.""" + # Check cache first + cache_path = self.cache_dir / target_id + cached_file = cache_path / "target" + + if cached_file.exists(): + # Update access time for LRU + cached_file.touch() + logger.info(f"Cache HIT: {target_id}") + return cached_file + + # Cache miss - download from S3 + logger.info(f"Cache MISS: {target_id}, downloading from S3...") + + try: + # Create cache directory + cache_path.mkdir(parents=True, exist_ok=True) + + # Download from S3 + s3_key = f'{target_id}/target' + logger.info(f"Downloading s3://{self.bucket}/{s3_key}") + + self.s3_client.download_file( + self.bucket, + s3_key, + str(cached_file) + ) + + # Verify download + if not cached_file.exists(): + raise StorageError(f"Downloaded file not found: {cached_file}") + + file_size_mb = cached_file.stat().st_size / (1024 * 1024) + logger.info(f"āœ“ Downloaded target {target_id} ({file_size_mb:.2f} MB)") + + return cached_file + + except ClientError as e: + error_code = e.response.get('Error', {}).get('Code') + if error_code in ['404', 'NoSuchKey']: + logger.error(f"Target not found: {target_id}") + raise FileNotFoundError(f"Target {target_id} not found in storage") + else: + logger.error(f"S3 download failed: {e}", exc_info=True) + raise StorageError(f"Download failed: {e}") + except Exception as e: + logger.error(f"Download error: {e}", exc_info=True) + # Cleanup partial download + if cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) + raise StorageError(f"Download error: {e}") + + async def delete_target(self, target_id: str) -> None: + """Delete target from S3/MinIO.""" + try: + s3_key = f'{target_id}/target' + logger.info(f"Deleting s3://{self.bucket}/{s3_key}") + + self.s3_client.delete_object( + Bucket=self.bucket, + Key=s3_key + ) + + # Also delete from cache if present + cache_path = self.cache_dir / target_id + if cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) + logger.info(f"āœ“ Deleted target {target_id} from S3 and cache") + else: + logger.info(f"āœ“ Deleted target {target_id} from S3") + + except ClientError as e: + logger.error(f"S3 delete failed: {e}", exc_info=True) + # Don't raise error if object doesn't exist + if e.response.get('Error', {}).get('Code') not in ['404', 'NoSuchKey']: + raise StorageError(f"Delete failed: {e}") + except Exception as e: + logger.error(f"Delete error: {e}", exc_info=True) + raise StorageError(f"Delete error: {e}") + + async def upload_results( + self, + workflow_id: str, + results: Dict[str, Any], + results_format: str = "json" + ) -> str: + """Upload workflow results to S3/MinIO.""" + try: + # Prepare results content + if results_format == "json": + content = json.dumps(results, indent=2).encode('utf-8') + content_type = 'application/json' + file_ext = 'json' + elif results_format == "sarif": + content = json.dumps(results, indent=2).encode('utf-8') + content_type = 'application/sarif+json' + file_ext = 'sarif' + else: + content = json.dumps(results, indent=2).encode('utf-8') + content_type = 'application/json' + file_ext = 'json' + + # Upload to results bucket + results_bucket = 'results' + s3_key = f'{workflow_id}/results.{file_ext}' + + logger.info(f"Uploading results to s3://{results_bucket}/{s3_key}") + + self.s3_client.put_object( + Bucket=results_bucket, + Key=s3_key, + Body=content, + ContentType=content_type, + Metadata={ + 'workflow_id': workflow_id, + 'format': results_format, + 'uploaded_at': datetime.now().isoformat() + } + ) + + # Construct URL + results_url = f"{self.endpoint_url}/{results_bucket}/{s3_key}" + logger.info(f"āœ“ Uploaded results: {results_url}") + + return results_url + + except Exception as e: + logger.error(f"Results upload failed: {e}", exc_info=True) + raise StorageError(f"Results upload failed: {e}") + + async def get_results(self, workflow_id: str) -> Dict[str, Any]: + """Get workflow results from S3/MinIO.""" + try: + results_bucket = 'results' + s3_key = f'{workflow_id}/results.json' + + logger.info(f"Downloading results from s3://{results_bucket}/{s3_key}") + + response = self.s3_client.get_object( + Bucket=results_bucket, + Key=s3_key + ) + + content = response['Body'].read().decode('utf-8') + results = json.loads(content) + + logger.info(f"āœ“ Downloaded results for workflow {workflow_id}") + return results + + except ClientError as e: + error_code = e.response.get('Error', {}).get('Code') + if error_code in ['404', 'NoSuchKey']: + logger.error(f"Results not found: {workflow_id}") + raise FileNotFoundError(f"Results for workflow {workflow_id} not found") + else: + logger.error(f"Results download failed: {e}", exc_info=True) + raise StorageError(f"Results download failed: {e}") + except Exception as e: + logger.error(f"Results download error: {e}", exc_info=True) + raise StorageError(f"Results download error: {e}") + + async def list_targets( + self, + user_id: Optional[str] = None, + limit: int = 100 + ) -> list[Dict[str, Any]]: + """List uploaded targets.""" + try: + targets = [] + paginator = self.s3_client.get_paginator('list_objects_v2') + + for page in paginator.paginate(Bucket=self.bucket, PaginationConfig={'MaxItems': limit}): + for obj in page.get('Contents', []): + # Get object metadata + try: + metadata_response = self.s3_client.head_object( + Bucket=self.bucket, + Key=obj['Key'] + ) + metadata = metadata_response.get('Metadata', {}) + + # Filter by user_id if specified + if user_id and metadata.get('user_id') != user_id: + continue + + targets.append({ + 'target_id': obj['Key'].split('/')[0], + 'key': obj['Key'], + 'size': obj['Size'], + 'last_modified': obj['LastModified'].isoformat(), + 'metadata': metadata + }) + + except Exception as e: + logger.warning(f"Failed to get metadata for {obj['Key']}: {e}") + continue + + logger.info(f"Listed {len(targets)} targets (user_id={user_id})") + return targets + + except Exception as e: + logger.error(f"List targets failed: {e}", exc_info=True) + raise StorageError(f"List targets failed: {e}") + + async def cleanup_cache(self) -> int: + """Clean up local cache using LRU eviction.""" + try: + cache_files = [] + total_size = 0 + + # Gather all cached files with metadata + for cache_file in self.cache_dir.rglob('*'): + if cache_file.is_file(): + try: + stat = cache_file.stat() + cache_files.append({ + 'path': cache_file, + 'size': stat.st_size, + 'atime': stat.st_atime # Last access time + }) + total_size += stat.st_size + except Exception as e: + logger.warning(f"Failed to stat {cache_file}: {e}") + continue + + # Check if cleanup is needed + if total_size <= self.cache_max_size: + logger.info( + f"Cache size OK: {total_size / (1024**3):.2f} GB / " + f"{self.cache_max_size / (1024**3):.2f} GB" + ) + return 0 + + # Sort by access time (oldest first) + cache_files.sort(key=lambda x: x['atime']) + + # Remove files until under limit + removed_count = 0 + for file_info in cache_files: + if total_size <= self.cache_max_size: + break + + try: + file_info['path'].unlink() + total_size -= file_info['size'] + removed_count += 1 + logger.debug(f"Evicted from cache: {file_info['path']}") + except Exception as e: + logger.warning(f"Failed to delete {file_info['path']}: {e}") + continue + + logger.info( + f"āœ“ Cache cleanup: removed {removed_count} files, " + f"new size: {total_size / (1024**3):.2f} GB" + ) + return removed_count + + except Exception as e: + logger.error(f"Cache cleanup failed: {e}", exc_info=True) + raise StorageError(f"Cache cleanup failed: {e}") + + def get_cache_stats(self) -> Dict[str, Any]: + """Get cache statistics.""" + try: + total_size = 0 + file_count = 0 + + for cache_file in self.cache_dir.rglob('*'): + if cache_file.is_file(): + total_size += cache_file.stat().st_size + file_count += 1 + + return { + 'total_size_bytes': total_size, + 'total_size_gb': total_size / (1024 ** 3), + 'file_count': file_count, + 'max_size_gb': self.cache_max_size / (1024 ** 3), + 'usage_percent': (total_size / self.cache_max_size) * 100 + } + except Exception as e: + logger.error(f"Failed to get cache stats: {e}") + return {'error': str(e)} diff --git a/backend/src/temporal/__init__.py b/backend/src/temporal/__init__.py new file mode 100644 index 0000000..acaa368 --- /dev/null +++ b/backend/src/temporal/__init__.py @@ -0,0 +1,10 @@ +""" +Temporal integration for FuzzForge. + +Handles workflow execution, monitoring, and management. +""" + +from .manager import TemporalManager +from .discovery import WorkflowDiscovery + +__all__ = ["TemporalManager", "WorkflowDiscovery"] diff --git a/backend/src/temporal/discovery.py b/backend/src/temporal/discovery.py new file mode 100644 index 0000000..07da6f8 --- /dev/null +++ b/backend/src/temporal/discovery.py @@ -0,0 +1,257 @@ +""" +Workflow Discovery for Temporal + +Discovers workflows from the toolbox/workflows directory +and provides metadata about available workflows. +""" + +import logging +import yaml +from pathlib import Path +from typing import Dict, Any +from pydantic import BaseModel, Field, ConfigDict + +logger = logging.getLogger(__name__) + + +class WorkflowInfo(BaseModel): + """Information about a discovered workflow""" + name: str = Field(..., description="Workflow name") + path: Path = Field(..., description="Path to workflow directory") + workflow_file: Path = Field(..., description="Path to workflow.py file") + metadata: Dict[str, Any] = Field(..., description="Workflow metadata from YAML") + workflow_type: str = Field(..., description="Workflow class name") + vertical: str = Field(..., description="Vertical (worker type) for this workflow") + + model_config = ConfigDict(arbitrary_types_allowed=True) + + +class WorkflowDiscovery: + """ + Discovers workflows from the filesystem. + + Scans toolbox/workflows/ for directories containing: + - metadata.yaml (required) + - workflow.py (required) + + Each workflow declares its vertical (rust, android, web, etc.) + which determines which worker pool will execute it. + """ + + def __init__(self, workflows_dir: Path): + """ + Initialize workflow discovery. + + Args: + workflows_dir: Path to the workflows directory + """ + self.workflows_dir = workflows_dir + if not self.workflows_dir.exists(): + self.workflows_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Created workflows directory: {self.workflows_dir}") + + async def discover_workflows(self) -> Dict[str, WorkflowInfo]: + """ + Discover workflows by scanning the workflows directory. + + Returns: + Dictionary mapping workflow names to their information + """ + workflows = {} + + logger.info(f"Scanning for workflows in: {self.workflows_dir}") + + for workflow_dir in self.workflows_dir.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + metadata_file = workflow_dir / "metadata.yaml" + if not metadata_file.exists(): + logger.debug(f"No metadata.yaml in {workflow_dir.name}, skipping") + continue + + workflow_file = workflow_dir / "workflow.py" + if not workflow_file.exists(): + logger.warning( + f"Workflow {workflow_dir.name} has metadata but no workflow.py, skipping" + ) + continue + + try: + # Parse metadata + with open(metadata_file) as f: + metadata = yaml.safe_load(f) + + # Validate required fields + if 'name' not in metadata: + logger.warning(f"Workflow {workflow_dir.name} metadata missing 'name' field") + metadata['name'] = workflow_dir.name + + if 'vertical' not in metadata: + logger.warning( + f"Workflow {workflow_dir.name} metadata missing 'vertical' field" + ) + continue + + # Infer workflow class name from metadata or use convention + workflow_type = metadata.get('workflow_class') + if not workflow_type: + # Convention: convert snake_case to PascalCase + Workflow + # e.g., rust_test -> RustTestWorkflow + parts = workflow_dir.name.split('_') + workflow_type = ''.join(part.capitalize() for part in parts) + 'Workflow' + + # Create workflow info + info = WorkflowInfo( + name=metadata['name'], + path=workflow_dir, + workflow_file=workflow_file, + metadata=metadata, + workflow_type=workflow_type, + vertical=metadata['vertical'] + ) + + workflows[info.name] = info + logger.info( + f"āœ“ Discovered workflow: {info.name} " + f"(vertical: {info.vertical}, class: {info.workflow_type})" + ) + + except Exception as e: + logger.error( + f"Error discovering workflow {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(workflows)} workflows") + return workflows + + def get_workflows_by_vertical( + self, + workflows: Dict[str, WorkflowInfo], + vertical: str + ) -> Dict[str, WorkflowInfo]: + """ + Filter workflows by vertical. + + Args: + workflows: All discovered workflows + vertical: Vertical name to filter by + + Returns: + Filtered workflows dictionary + """ + return { + name: info + for name, info in workflows.items() + if info.vertical == vertical + } + + def get_available_verticals(self, workflows: Dict[str, WorkflowInfo]) -> list[str]: + """ + Get list of all verticals from discovered workflows. + + Args: + workflows: All discovered workflows + + Returns: + List of unique vertical names + """ + return list(set(info.vertical for info in workflows.values())) + + @staticmethod + def get_metadata_schema() -> Dict[str, Any]: + """ + Get the JSON schema for workflow metadata. + + Returns: + JSON schema dictionary + """ + return { + "type": "object", + "required": ["name", "version", "description", "author", "vertical", "parameters"], + "properties": { + "name": { + "type": "string", + "description": "Workflow name" + }, + "version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$", + "description": "Semantic version (x.y.z)" + }, + "vertical": { + "type": "string", + "description": "Vertical worker type (rust, android, web, etc.)" + }, + "description": { + "type": "string", + "description": "Workflow description" + }, + "author": { + "type": "string", + "description": "Workflow author" + }, + "category": { + "type": "string", + "enum": ["comprehensive", "specialized", "fuzzing", "focused"], + "description": "Workflow category" + }, + "tags": { + "type": "array", + "items": {"type": "string"}, + "description": "Workflow tags for categorization" + }, + "requirements": { + "type": "object", + "required": ["tools", "resources"], + "properties": { + "tools": { + "type": "array", + "items": {"type": "string"}, + "description": "Required security tools" + }, + "resources": { + "type": "object", + "required": ["memory", "cpu", "timeout"], + "properties": { + "memory": { + "type": "string", + "pattern": "^\\d+[GMK]i$", + "description": "Memory limit (e.g., 1Gi, 512Mi)" + }, + "cpu": { + "type": "string", + "pattern": "^\\d+m?$", + "description": "CPU limit (e.g., 1000m, 2)" + }, + "timeout": { + "type": "integer", + "minimum": 60, + "maximum": 7200, + "description": "Workflow timeout in seconds" + } + } + } + } + }, + "parameters": { + "type": "object", + "description": "Workflow parameters schema" + }, + "default_parameters": { + "type": "object", + "description": "Default parameter values" + }, + "required_modules": { + "type": "array", + "items": {"type": "string"}, + "description": "Required module names" + } + } + } diff --git a/backend/src/temporal/manager.py b/backend/src/temporal/manager.py new file mode 100644 index 0000000..9a44e8b --- /dev/null +++ b/backend/src/temporal/manager.py @@ -0,0 +1,376 @@ +""" +Temporal Manager - Workflow execution and management + +Handles: +- Workflow discovery from toolbox +- Workflow execution (submit to Temporal) +- Status monitoring +- Results retrieval +""" + +import logging +import os +from pathlib import Path +from typing import Dict, Optional, Any +from uuid import uuid4 + +from temporalio.client import Client, WorkflowHandle +from temporalio.common import RetryPolicy +from datetime import timedelta + +from .discovery import WorkflowDiscovery, WorkflowInfo +from src.storage import S3CachedStorage + +logger = logging.getLogger(__name__) + + +class TemporalManager: + """ + Manages Temporal workflow execution for FuzzForge. + + This class: + - Discovers available workflows from toolbox + - Submits workflow executions to Temporal + - Monitors workflow status + - Retrieves workflow results + """ + + def __init__( + self, + workflows_dir: Optional[Path] = None, + temporal_address: Optional[str] = None, + temporal_namespace: str = "default", + storage: Optional[S3CachedStorage] = None + ): + """ + Initialize Temporal manager. + + Args: + workflows_dir: Path to workflows directory (default: toolbox/workflows) + temporal_address: Temporal server address (default: from env or localhost:7233) + temporal_namespace: Temporal namespace + storage: Storage backend for file uploads (default: S3CachedStorage) + """ + if workflows_dir is None: + workflows_dir = Path("toolbox/workflows") + + self.temporal_address = temporal_address or os.getenv( + 'TEMPORAL_ADDRESS', + 'localhost:7233' + ) + self.temporal_namespace = temporal_namespace + self.discovery = WorkflowDiscovery(workflows_dir) + self.workflows: Dict[str, WorkflowInfo] = {} + self.client: Optional[Client] = None + + # Initialize storage backend + self.storage = storage or S3CachedStorage() + + logger.info( + f"TemporalManager initialized: {self.temporal_address} " + f"(namespace: {self.temporal_namespace})" + ) + + async def initialize(self): + """Initialize the manager by discovering workflows and connecting to Temporal.""" + try: + # Discover workflows + self.workflows = await self.discovery.discover_workflows() + + if not self.workflows: + logger.warning("No workflows discovered") + else: + logger.info( + f"Discovered {len(self.workflows)} workflows: " + f"{list(self.workflows.keys())}" + ) + + # Connect to Temporal + self.client = await Client.connect( + self.temporal_address, + namespace=self.temporal_namespace + ) + logger.info(f"āœ“ Connected to Temporal: {self.temporal_address}") + + except Exception as e: + logger.error(f"Failed to initialize Temporal manager: {e}", exc_info=True) + raise + + async def close(self): + """Close Temporal client connection.""" + if self.client: + # Temporal client doesn't need explicit close in Python SDK + pass + + async def get_workflows(self) -> Dict[str, WorkflowInfo]: + """ + Get all discovered workflows. + + Returns: + Dictionary mapping workflow names to their info + """ + return self.workflows + + async def get_workflow(self, name: str) -> Optional[WorkflowInfo]: + """ + Get workflow info by name. + + Args: + name: Workflow name + + Returns: + WorkflowInfo or None if not found + """ + return self.workflows.get(name) + + async def upload_target( + self, + file_path: Path, + user_id: str, + metadata: Optional[Dict[str, Any]] = None + ) -> str: + """ + Upload target file to storage. + + Args: + file_path: Local path to file + user_id: User ID + metadata: Optional metadata + + Returns: + Target ID for use in workflow execution + """ + target_id = await self.storage.upload_target(file_path, user_id, metadata) + logger.info(f"Uploaded target: {target_id}") + return target_id + + async def run_workflow( + self, + workflow_name: str, + target_id: str, + workflow_params: Optional[Dict[str, Any]] = None, + workflow_id: Optional[str] = None + ) -> WorkflowHandle: + """ + Execute a workflow. + + Args: + workflow_name: Name of workflow to execute + target_id: Target ID (from upload_target) + workflow_params: Additional workflow parameters + workflow_id: Optional workflow ID (generated if not provided) + + Returns: + WorkflowHandle for monitoring/results + + Raises: + ValueError: If workflow not found or client not initialized + """ + if not self.client: + raise ValueError("Temporal client not initialized. Call initialize() first.") + + # Get workflow info + workflow_info = self.workflows.get(workflow_name) + if not workflow_info: + raise ValueError(f"Workflow not found: {workflow_name}") + + # Generate workflow ID if not provided + if not workflow_id: + workflow_id = f"{workflow_name}-{str(uuid4())[:8]}" + + # Prepare workflow input arguments + workflow_params = workflow_params or {} + + # Build args list: [target_id, ...workflow_params in schema order] + # The workflow parameters are passed as individual positional args + workflow_args = [target_id] + + # Add parameters in order based on metadata schema + # This ensures parameters match the workflow signature order + if workflow_params and 'parameters' in workflow_info.metadata: + param_schema = workflow_info.metadata['parameters'].get('properties', {}) + # Iterate parameters in schema order and add values + for param_name in param_schema.keys(): + param_value = workflow_params.get(param_name) + workflow_args.append(param_value) + + # Determine task queue from workflow vertical + vertical = workflow_info.metadata.get("vertical", "default") + task_queue = f"{vertical}-queue" + + logger.info( + f"Starting workflow: {workflow_name} " + f"(id={workflow_id}, queue={task_queue}, target={target_id})" + ) + logger.info(f"DEBUG: workflow_args = {workflow_args}") + logger.info(f"DEBUG: workflow_params received = {workflow_params}") + + try: + # Start workflow execution with positional arguments + handle = await self.client.start_workflow( + workflow=workflow_info.workflow_type, # Workflow class name + args=workflow_args, # Positional arguments + id=workflow_id, + task_queue=task_queue, + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(minutes=1), + maximum_attempts=3 + ) + ) + + logger.info(f"āœ“ Workflow started: {workflow_id}") + return handle + + except Exception as e: + logger.error(f"Failed to start workflow {workflow_name}: {e}", exc_info=True) + raise + + async def get_workflow_status(self, workflow_id: str) -> Dict[str, Any]: + """ + Get workflow execution status. + + Args: + workflow_id: Workflow execution ID + + Returns: + Status dictionary with workflow state + + Raises: + ValueError: If client not initialized or workflow not found + """ + if not self.client: + raise ValueError("Temporal client not initialized") + + try: + # Get workflow handle + handle = self.client.get_workflow_handle(workflow_id) + + # Try to get result (non-blocking describe) + description = await handle.describe() + + status = { + "workflow_id": workflow_id, + "status": description.status.name, + "start_time": description.start_time.isoformat() if description.start_time else None, + "execution_time": description.execution_time.isoformat() if description.execution_time else None, + "close_time": description.close_time.isoformat() if description.close_time else None, + "task_queue": description.task_queue, + } + + logger.info(f"Workflow {workflow_id} status: {status['status']}") + return status + + except Exception as e: + logger.error(f"Failed to get workflow status: {e}", exc_info=True) + raise + + async def get_workflow_result( + self, + workflow_id: str, + timeout: Optional[timedelta] = None + ) -> Any: + """ + Get workflow execution result (blocking). + + Args: + workflow_id: Workflow execution ID + timeout: Maximum time to wait for result + + Returns: + Workflow result + + Raises: + ValueError: If client not initialized + TimeoutError: If timeout exceeded + """ + if not self.client: + raise ValueError("Temporal client not initialized") + + try: + handle = self.client.get_workflow_handle(workflow_id) + + logger.info(f"Waiting for workflow result: {workflow_id}") + + # Wait for workflow to complete and get result + if timeout: + # Use asyncio timeout if provided + import asyncio + result = await asyncio.wait_for(handle.result(), timeout=timeout.total_seconds()) + else: + result = await handle.result() + + logger.info(f"āœ“ Workflow {workflow_id} completed") + return result + + except Exception as e: + logger.error(f"Failed to get workflow result: {e}", exc_info=True) + raise + + async def cancel_workflow(self, workflow_id: str) -> None: + """ + Cancel a running workflow. + + Args: + workflow_id: Workflow execution ID + + Raises: + ValueError: If client not initialized + """ + if not self.client: + raise ValueError("Temporal client not initialized") + + try: + handle = self.client.get_workflow_handle(workflow_id) + await handle.cancel() + + logger.info(f"āœ“ Workflow cancelled: {workflow_id}") + + except Exception as e: + logger.error(f"Failed to cancel workflow: {e}", exc_info=True) + raise + + async def list_workflows( + self, + filter_query: Optional[str] = None, + limit: int = 100 + ) -> list[Dict[str, Any]]: + """ + List workflow executions. + + Args: + filter_query: Optional Temporal list filter query + limit: Maximum number of results + + Returns: + List of workflow execution info + + Raises: + ValueError: If client not initialized + """ + if not self.client: + raise ValueError("Temporal client not initialized") + + try: + workflows = [] + + # Use Temporal's list API + async for workflow in self.client.list_workflows(filter_query): + workflows.append({ + "workflow_id": workflow.id, + "workflow_type": workflow.workflow_type, + "status": workflow.status.name, + "start_time": workflow.start_time.isoformat() if workflow.start_time else None, + "close_time": workflow.close_time.isoformat() if workflow.close_time else None, + "task_queue": workflow.task_queue, + }) + + if len(workflows) >= limit: + break + + logger.info(f"Listed {len(workflows)} workflows") + return workflows + + except Exception as e: + logger.error(f"Failed to list workflows: {e}", exc_info=True) + raise diff --git a/backend/tests/README.md b/backend/tests/README.md new file mode 100644 index 0000000..a1cada4 --- /dev/null +++ b/backend/tests/README.md @@ -0,0 +1,119 @@ +# FuzzForge Test Suite + +Comprehensive test infrastructure for FuzzForge modules and workflows. + +## Directory Structure + +``` +tests/ +ā”œā”€ā”€ conftest.py # Shared pytest fixtures +ā”œā”€ā”€ unit/ # Fast, isolated unit tests +ā”‚ ā”œā”€ā”€ test_modules/ # Module-specific tests +ā”‚ ā”‚ ā”œā”€ā”€ test_cargo_fuzzer.py +ā”‚ ā”‚ ā””ā”€ā”€ test_atheris_fuzzer.py +ā”‚ ā”œā”€ā”€ test_workflows/ # Workflow tests +ā”‚ ā””ā”€ā”€ test_api/ # API endpoint tests +ā”œā”€ā”€ integration/ # Integration tests (requires Docker) +ā””ā”€ā”€ fixtures/ # Test data and projects + ā”œā”€ā”€ test_projects/ # Vulnerable projects for testing + ā””ā”€ā”€ expected_results/ # Expected output for validation +``` + +## Running Tests + +### All Tests +```bash +cd backend +pytest tests/ -v +``` + +### Unit Tests Only (Fast) +```bash +pytest tests/unit/ -v +``` + +### Integration Tests (Requires Docker) +```bash +# Start services +docker-compose up -d + +# Run integration tests +pytest tests/integration/ -v + +# Cleanup +docker-compose down +``` + +### With Coverage +```bash +pytest tests/ --cov=toolbox/modules --cov=src --cov-report=html +``` + +### Parallel Execution +```bash +pytest tests/unit/ -n auto +``` + +## Available Fixtures + +### Workspace Fixtures +- `temp_workspace`: Empty temporary workspace +- `python_test_workspace`: Python project with vulnerabilities +- `rust_test_workspace`: Rust project with fuzz targets + +### Module Fixtures +- `atheris_fuzzer`: AtherisFuzzer instance +- `cargo_fuzzer`: CargoFuzzer instance +- `file_scanner`: FileScanner instance + +### Configuration Fixtures +- `atheris_config`: Default Atheris configuration +- `cargo_fuzz_config`: Default cargo-fuzz configuration +- `gitleaks_config`: Default Gitleaks configuration + +### Mock Fixtures +- `mock_stats_callback`: Mock stats callback for fuzzing +- `mock_temporal_context`: Mock Temporal activity context + +## Writing Tests + +### Unit Test Example +```python +import pytest + +@pytest.mark.asyncio +async def test_module_execution(cargo_fuzzer, rust_test_workspace, cargo_fuzz_config): + """Test module execution""" + result = await cargo_fuzzer.execute(cargo_fuzz_config, rust_test_workspace) + + assert result.status == "success" + assert result.execution_time > 0 +``` + +### Integration Test Example +```python +@pytest.mark.integration +async def test_end_to_end_workflow(): + """Test complete workflow execution""" + # Test full workflow with real services + pass +``` + +## CI/CD Integration + +Tests run automatically on: +- **Push to main/develop**: Full test suite +- **Pull requests**: Full test suite + coverage +- **Nightly**: Extended integration tests + +See `.github/workflows/test.yml` for configuration. + +## Code Coverage + +Target coverage: **80%+** for core modules + +View coverage report: +```bash +pytest tests/ --cov --cov-report=html +open htmlcov/index.html +``` diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 7ab7ec3..0bc6eee 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -11,9 +11,220 @@ import sys from pathlib import Path +from typing import Dict, Any +import pytest # Ensure project root is on sys.path so `src` is importable ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +# Add toolbox to path for module imports +TOOLBOX = ROOT / "toolbox" +if str(TOOLBOX) not in sys.path: + sys.path.insert(0, str(TOOLBOX)) + + +# ============================================================================ +# Workspace Fixtures +# ============================================================================ + +@pytest.fixture +def temp_workspace(tmp_path): + """Create a temporary workspace directory for testing""" + workspace = tmp_path / "workspace" + workspace.mkdir() + return workspace + + +@pytest.fixture +def python_test_workspace(temp_workspace): + """Create a Python test workspace with sample files""" + # Create a simple Python project structure + (temp_workspace / "main.py").write_text(""" +def process_data(data): + # Intentional bug: no bounds checking + return data[0:100] + +def divide(a, b): + # Division by zero vulnerability + return a / b +""") + + (temp_workspace / "config.py").write_text(""" +# Hardcoded secrets for testing +API_KEY = "sk_test_1234567890abcdef" +DATABASE_URL = "postgresql://admin:password123@localhost/db" +AWS_SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +""") + + return temp_workspace + + +@pytest.fixture +def rust_test_workspace(temp_workspace): + """Create a Rust test workspace with fuzz targets""" + # Create Cargo.toml + (temp_workspace / "Cargo.toml").write_text("""[package] +name = "test_project" +version = "0.1.0" +edition = "2021" + +[dependencies] +""") + + # Create src/lib.rs + src_dir = temp_workspace / "src" + src_dir.mkdir() + (src_dir / "lib.rs").write_text(""" +pub fn process_buffer(data: &[u8]) -> Vec { + if data.len() < 4 { + return Vec::new(); + } + + // Vulnerability: bounds checking issue + let size = data[0] as usize; + let mut result = Vec::new(); + for i in 0..size { + result.push(data[i]); + } + result +} +""") + + # Create fuzz directory structure + fuzz_dir = temp_workspace / "fuzz" + fuzz_dir.mkdir() + + (fuzz_dir / "Cargo.toml").write_text("""[package] +name = "test_project-fuzz" +version = "0.0.0" +edition = "2021" + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.test_project] +path = ".." + +[[bin]] +name = "fuzz_target_1" +path = "fuzz_targets/fuzz_target_1.rs" +""") + + fuzz_targets_dir = fuzz_dir / "fuzz_targets" + fuzz_targets_dir.mkdir() + + (fuzz_targets_dir / "fuzz_target_1.rs").write_text("""#![no_main] +use libfuzzer_sys::fuzz_target; +use test_project::process_buffer; + +fuzz_target!(|data: &[u8]| { + let _ = process_buffer(data); +}); +""") + + return temp_workspace + + +# ============================================================================ +# Module Configuration Fixtures +# ============================================================================ + +@pytest.fixture +def atheris_config(): + """Default Atheris fuzzer configuration""" + return { + "target_file": "auto-discover", + "max_iterations": 1000, + "timeout_seconds": 10, + "corpus_dir": None + } + + +@pytest.fixture +def cargo_fuzz_config(): + """Default cargo-fuzz configuration""" + return { + "target_name": None, + "max_iterations": 1000, + "timeout_seconds": 10, + "sanitizer": "address" + } + + +@pytest.fixture +def gitleaks_config(): + """Default Gitleaks configuration""" + return { + "config_path": None, + "scan_uncommitted": True + } + + +@pytest.fixture +def file_scanner_config(): + """Default file scanner configuration""" + return { + "scan_patterns": ["*.py", "*.rs", "*.js"], + "exclude_patterns": ["*.test.*", "*.spec.*"], + "max_file_size": 1048576 # 1MB + } + + +# ============================================================================ +# Module Instance Fixtures +# ============================================================================ + +@pytest.fixture +def atheris_fuzzer(): + """Create an AtherisFuzzer instance""" + from modules.fuzzer.atheris_fuzzer import AtherisFuzzer + return AtherisFuzzer() + + +@pytest.fixture +def cargo_fuzzer(): + """Create a CargoFuzzer instance""" + from modules.fuzzer.cargo_fuzzer import CargoFuzzer + return CargoFuzzer() + + +@pytest.fixture +def file_scanner(): + """Create a FileScanner instance""" + from modules.scanner.file_scanner import FileScanner + return FileScanner() + + +# ============================================================================ +# Mock Fixtures +# ============================================================================ + +@pytest.fixture +def mock_stats_callback(): + """Mock stats callback for fuzzing""" + stats_received = [] + + async def callback(stats: Dict[str, Any]): + stats_received.append(stats) + + callback.stats_received = stats_received + return callback + + +@pytest.fixture +def mock_temporal_context(): + """Mock Temporal activity context""" + class MockActivityInfo: + def __init__(self): + self.workflow_id = "test-workflow-123" + self.activity_id = "test-activity-1" + self.attempt = 1 + + class MockContext: + def __init__(self): + self.info = MockActivityInfo() + + return MockContext() + diff --git a/backend/tests/fixtures/__init__.py b/backend/tests/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/integration/__init__.py b/backend/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_prefect_stats_monitor.py b/backend/tests/test_prefect_stats_monitor.py deleted file mode 100644 index 16c29df..0000000 --- a/backend/tests/test_prefect_stats_monitor.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - -import asyncio -from datetime import datetime, timezone, timedelta - - -from src.services.prefect_stats_monitor import PrefectStatsMonitor -from src.api import fuzzing - - -class FakeLog: - def __init__(self, message: str): - self.message = message - - -class FakeClient: - def __init__(self, logs): - self._logs = logs - - async def read_logs(self, log_filter=None, limit=100, sort="TIMESTAMP_ASC"): - return self._logs - - -class FakeTaskRun: - def __init__(self): - self.id = "task-1" - self.start_time = datetime.now(timezone.utc) - timedelta(seconds=5) - - -def test_parse_stats_from_log_fuzzing(): - mon = PrefectStatsMonitor() - msg = ( - "INFO LIVE_STATS extra={'stats_type': 'fuzzing_live_update', " - "'executions': 42, 'executions_per_sec': 3.14, 'crashes': 1, 'unique_crashes': 1, 'corpus_size': 9}" - ) - stats = mon._parse_stats_from_log(msg) - assert stats is not None - assert stats["stats_type"] == "fuzzing_live_update" - assert stats["executions"] == 42 - - -def test_extract_stats_updates_and_broadcasts(): - mon = PrefectStatsMonitor() - run_id = "run-123" - workflow = "wf" - fuzzing.initialize_fuzzing_tracking(run_id, workflow) - - # Prepare a fake websocket to capture messages - sent = [] - - class FakeWS: - async def send_text(self, text: str): - sent.append(text) - - fuzzing.active_connections[run_id] = [FakeWS()] - - # Craft a log line the parser understands - msg = ( - "INFO LIVE_STATS extra={'stats_type': 'fuzzing_live_update', " - "'executions': 10, 'executions_per_sec': 1.5, 'crashes': 0, 'unique_crashes': 0, 'corpus_size': 2}" - ) - fake_client = FakeClient([FakeLog(msg)]) - task_run = FakeTaskRun() - - asyncio.run(mon._extract_stats_from_task(fake_client, run_id, task_run, workflow)) - - # Verify stats updated - stats = fuzzing.fuzzing_stats[run_id] - assert stats.executions == 10 - assert stats.executions_per_sec == 1.5 - - # Verify a message was sent to WebSocket - assert sent, "Expected a stats_update message to be sent" diff --git a/backend/tests/unit/__init__.py b/backend/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/test_api/__init__.py b/backend/tests/unit/test_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/test_modules/__init__.py b/backend/tests/unit/test_modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/unit/test_modules/test_atheris_fuzzer.py b/backend/tests/unit/test_modules/test_atheris_fuzzer.py new file mode 100644 index 0000000..9cd01ce --- /dev/null +++ b/backend/tests/unit/test_modules/test_atheris_fuzzer.py @@ -0,0 +1,177 @@ +""" +Unit tests for AtherisFuzzer module +""" + +import pytest +from unittest.mock import AsyncMock, patch + + +@pytest.mark.asyncio +class TestAtherisFuzzerMetadata: + """Test AtherisFuzzer metadata""" + + async def test_metadata_structure(self, atheris_fuzzer): + """Test that module metadata is properly defined""" + metadata = atheris_fuzzer.get_metadata() + + assert metadata.name == "atheris_fuzzer" + assert metadata.category == "fuzzer" + assert "fuzzing" in metadata.tags + assert "python" in metadata.tags + + +@pytest.mark.asyncio +class TestAtherisFuzzerConfigValidation: + """Test configuration validation""" + + async def test_valid_config(self, atheris_fuzzer, atheris_config): + """Test validation of valid configuration""" + assert atheris_fuzzer.validate_config(atheris_config) is True + + async def test_invalid_max_iterations(self, atheris_fuzzer): + """Test validation fails with invalid max_iterations""" + config = { + "target_file": "fuzz_target.py", + "max_iterations": -1, + "timeout_seconds": 10 + } + with pytest.raises(ValueError, match="max_iterations"): + atheris_fuzzer.validate_config(config) + + async def test_invalid_timeout(self, atheris_fuzzer): + """Test validation fails with invalid timeout""" + config = { + "target_file": "fuzz_target.py", + "max_iterations": 1000, + "timeout_seconds": 0 + } + with pytest.raises(ValueError, match="timeout_seconds"): + atheris_fuzzer.validate_config(config) + + +@pytest.mark.asyncio +class TestAtherisFuzzerDiscovery: + """Test fuzz target discovery""" + + async def test_auto_discover(self, atheris_fuzzer, python_test_workspace): + """Test auto-discovery of Python fuzz targets""" + # Create a fuzz target file + (python_test_workspace / "fuzz_target.py").write_text(""" +import atheris +import sys + +def TestOneInput(data): + pass + +if __name__ == "__main__": + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() +""") + + # Pass None for auto-discovery + target = atheris_fuzzer._discover_target(python_test_workspace, None) + + assert target is not None + assert "fuzz_target.py" in str(target) + + +@pytest.mark.asyncio +class TestAtherisFuzzerExecution: + """Test fuzzer execution logic""" + + async def test_execution_creates_result(self, atheris_fuzzer, python_test_workspace, atheris_config): + """Test that execution returns a ModuleResult""" + # Create a simple fuzz target + (python_test_workspace / "fuzz_target.py").write_text(""" +import atheris +import sys + +def TestOneInput(data): + if len(data) > 0: + pass + +if __name__ == "__main__": + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() +""") + + # Use a very short timeout for testing + test_config = { + "target_file": "fuzz_target.py", + "max_iterations": 10, + "timeout_seconds": 1 + } + + # Mock the fuzzing subprocess to avoid actual execution + with patch.object(atheris_fuzzer, '_run_fuzzing', new_callable=AsyncMock, return_value=([], {"total_executions": 10})): + result = await atheris_fuzzer.execute(test_config, python_test_workspace) + + assert result.module == "atheris_fuzzer" + assert result.status in ["success", "partial", "failed"] + assert isinstance(result.execution_time, float) + + +@pytest.mark.asyncio +class TestAtherisFuzzerStatsCallback: + """Test stats callback functionality""" + + async def test_stats_callback_invoked(self, atheris_fuzzer, python_test_workspace, atheris_config, mock_stats_callback): + """Test that stats callback is invoked during fuzzing""" + (python_test_workspace / "fuzz_target.py").write_text(""" +import atheris +import sys + +def TestOneInput(data): + pass + +if __name__ == "__main__": + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() +""") + + # Mock fuzzing to simulate stats + async def mock_run_fuzzing(test_one_input, target_path, workspace, max_iterations, timeout_seconds, stats_callback): + if stats_callback: + await stats_callback({ + "total_execs": 100, + "execs_per_sec": 10.0, + "crashes": 0, + "coverage": 5, + "corpus_size": 2, + "elapsed_time": 10 + }) + return + + with patch.object(atheris_fuzzer, '_run_fuzzing', side_effect=mock_run_fuzzing): + with patch.object(atheris_fuzzer, '_load_target_module', return_value=lambda x: None): + # Put stats_callback in config dict, not as kwarg + atheris_config["target_file"] = "fuzz_target.py" + atheris_config["stats_callback"] = mock_stats_callback + await atheris_fuzzer.execute(atheris_config, python_test_workspace) + + # Verify callback was invoked + assert len(mock_stats_callback.stats_received) > 0 + + +@pytest.mark.asyncio +class TestAtherisFuzzerFindingGeneration: + """Test finding generation from crashes""" + + async def test_create_crash_finding(self, atheris_fuzzer): + """Test crash finding creation""" + finding = atheris_fuzzer.create_finding( + title="Crash: Exception in TestOneInput", + description="IndexError: list index out of range", + severity="high", + category="crash", + file_path="fuzz_target.py", + metadata={ + "crash_type": "IndexError", + "stack_trace": "Traceback..." + } + ) + + assert finding.title == "Crash: Exception in TestOneInput" + assert finding.severity == "high" + assert finding.category == "crash" + assert "IndexError" in finding.metadata["crash_type"] diff --git a/backend/tests/unit/test_modules/test_cargo_fuzzer.py b/backend/tests/unit/test_modules/test_cargo_fuzzer.py new file mode 100644 index 0000000..f550b9a --- /dev/null +++ b/backend/tests/unit/test_modules/test_cargo_fuzzer.py @@ -0,0 +1,177 @@ +""" +Unit tests for CargoFuzzer module +""" + +import pytest +from unittest.mock import AsyncMock, patch + + +@pytest.mark.asyncio +class TestCargoFuzzerMetadata: + """Test CargoFuzzer metadata""" + + async def test_metadata_structure(self, cargo_fuzzer): + """Test that module metadata is properly defined""" + metadata = cargo_fuzzer.get_metadata() + + assert metadata.name == "cargo_fuzz" + assert metadata.version == "0.11.2" + assert metadata.category == "fuzzer" + assert "fuzzing" in metadata.tags + assert "rust" in metadata.tags + + +@pytest.mark.asyncio +class TestCargoFuzzerConfigValidation: + """Test configuration validation""" + + async def test_valid_config(self, cargo_fuzzer, cargo_fuzz_config): + """Test validation of valid configuration""" + assert cargo_fuzzer.validate_config(cargo_fuzz_config) is True + + async def test_invalid_max_iterations(self, cargo_fuzzer): + """Test validation fails with invalid max_iterations""" + config = { + "max_iterations": -1, + "timeout_seconds": 10, + "sanitizer": "address" + } + with pytest.raises(ValueError, match="max_iterations"): + cargo_fuzzer.validate_config(config) + + async def test_invalid_timeout(self, cargo_fuzzer): + """Test validation fails with invalid timeout""" + config = { + "max_iterations": 1000, + "timeout_seconds": 0, + "sanitizer": "address" + } + with pytest.raises(ValueError, match="timeout_seconds"): + cargo_fuzzer.validate_config(config) + + async def test_invalid_sanitizer(self, cargo_fuzzer): + """Test validation fails with invalid sanitizer""" + config = { + "max_iterations": 1000, + "timeout_seconds": 10, + "sanitizer": "invalid_sanitizer" + } + with pytest.raises(ValueError, match="sanitizer"): + cargo_fuzzer.validate_config(config) + + +@pytest.mark.asyncio +class TestCargoFuzzerWorkspaceValidation: + """Test workspace validation""" + + async def test_valid_workspace(self, cargo_fuzzer, rust_test_workspace): + """Test validation of valid workspace""" + assert cargo_fuzzer.validate_workspace(rust_test_workspace) is True + + async def test_nonexistent_workspace(self, cargo_fuzzer, tmp_path): + """Test validation fails with nonexistent workspace""" + nonexistent = tmp_path / "does_not_exist" + with pytest.raises(ValueError, match="does not exist"): + cargo_fuzzer.validate_workspace(nonexistent) + + async def test_workspace_is_file(self, cargo_fuzzer, tmp_path): + """Test validation fails when workspace is a file""" + file_path = tmp_path / "file.txt" + file_path.write_text("test") + with pytest.raises(ValueError, match="not a directory"): + cargo_fuzzer.validate_workspace(file_path) + + +@pytest.mark.asyncio +class TestCargoFuzzerDiscovery: + """Test fuzz target discovery""" + + async def test_discover_targets(self, cargo_fuzzer, rust_test_workspace): + """Test discovery of fuzz targets""" + targets = await cargo_fuzzer._discover_fuzz_targets(rust_test_workspace) + + assert len(targets) == 1 + assert "fuzz_target_1" in targets + + async def test_no_fuzz_directory(self, cargo_fuzzer, temp_workspace): + """Test discovery with no fuzz directory""" + targets = await cargo_fuzzer._discover_fuzz_targets(temp_workspace) + + assert targets == [] + + +@pytest.mark.asyncio +class TestCargoFuzzerExecution: + """Test fuzzer execution logic""" + + async def test_execution_creates_result(self, cargo_fuzzer, rust_test_workspace, cargo_fuzz_config): + """Test that execution returns a ModuleResult""" + # Mock the build and run methods to avoid actual fuzzing + with patch.object(cargo_fuzzer, '_build_fuzz_target', new_callable=AsyncMock, return_value=True): + with patch.object(cargo_fuzzer, '_run_fuzzing', new_callable=AsyncMock, return_value=([], {"total_executions": 0, "crashes_found": 0})): + with patch.object(cargo_fuzzer, '_parse_crash_artifacts', new_callable=AsyncMock, return_value=[]): + result = await cargo_fuzzer.execute(cargo_fuzz_config, rust_test_workspace) + + assert result.module == "cargo_fuzz" + assert result.status == "success" + assert isinstance(result.execution_time, float) + assert result.execution_time >= 0 + + async def test_execution_with_no_targets(self, cargo_fuzzer, temp_workspace, cargo_fuzz_config): + """Test execution fails gracefully with no fuzz targets""" + result = await cargo_fuzzer.execute(cargo_fuzz_config, temp_workspace) + + assert result.status == "failed" + assert "No fuzz targets found" in result.error + + +@pytest.mark.asyncio +class TestCargoFuzzerStatsCallback: + """Test stats callback functionality""" + + async def test_stats_callback_invoked(self, cargo_fuzzer, rust_test_workspace, cargo_fuzz_config, mock_stats_callback): + """Test that stats callback is invoked during fuzzing""" + # Mock build/run to simulate stats generation + async def mock_run_fuzzing(workspace, target, config, callback): + # Simulate stats callback + if callback: + await callback({ + "total_execs": 1000, + "execs_per_sec": 100.0, + "crashes": 0, + "coverage": 10, + "corpus_size": 5, + "elapsed_time": 10 + }) + return [], {"total_executions": 1000} + + with patch.object(cargo_fuzzer, '_build_fuzz_target', new_callable=AsyncMock, return_value=True): + with patch.object(cargo_fuzzer, '_run_fuzzing', side_effect=mock_run_fuzzing): + with patch.object(cargo_fuzzer, '_parse_crash_artifacts', new_callable=AsyncMock, return_value=[]): + await cargo_fuzzer.execute(cargo_fuzz_config, rust_test_workspace, stats_callback=mock_stats_callback) + + # Verify callback was invoked + assert len(mock_stats_callback.stats_received) > 0 + assert mock_stats_callback.stats_received[0]["total_execs"] == 1000 + + +@pytest.mark.asyncio +class TestCargoFuzzerFindingGeneration: + """Test finding generation from crashes""" + + async def test_create_finding_from_crash(self, cargo_fuzzer): + """Test finding creation""" + finding = cargo_fuzzer.create_finding( + title="Crash: Segmentation Fault", + description="Test crash", + severity="critical", + category="crash", + file_path="fuzz/fuzz_targets/fuzz_target_1.rs", + metadata={"crash_type": "SIGSEGV"} + ) + + assert finding.title == "Crash: Segmentation Fault" + assert finding.severity == "critical" + assert finding.category == "crash" + assert finding.file_path == "fuzz/fuzz_targets/fuzz_target_1.rs" + assert finding.metadata["crash_type"] == "SIGSEGV" diff --git a/backend/tests/unit/test_modules/test_file_scanner.py b/backend/tests/unit/test_modules/test_file_scanner.py new file mode 100644 index 0000000..12332f0 --- /dev/null +++ b/backend/tests/unit/test_modules/test_file_scanner.py @@ -0,0 +1,349 @@ +""" +Unit tests for FileScanner module +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "toolbox")) + + + +@pytest.mark.asyncio +class TestFileScannerMetadata: + """Test FileScanner metadata""" + + async def test_metadata_structure(self, file_scanner): + """Test that metadata has correct structure""" + metadata = file_scanner.get_metadata() + + assert metadata.name == "file_scanner" + assert metadata.version == "1.0.0" + assert metadata.category == "scanner" + assert "files" in metadata.tags + assert "enumeration" in metadata.tags + assert metadata.requires_workspace is True + + +@pytest.mark.asyncio +class TestFileScannerConfigValidation: + """Test configuration validation""" + + async def test_valid_config(self, file_scanner): + """Test that valid config passes validation""" + config = { + "patterns": ["*.py", "*.js"], + "max_file_size": 1048576, + "check_sensitive": True, + "calculate_hashes": False + } + assert file_scanner.validate_config(config) is True + + async def test_default_config(self, file_scanner): + """Test that empty config uses defaults""" + config = {} + assert file_scanner.validate_config(config) is True + + async def test_invalid_patterns_type(self, file_scanner): + """Test that non-list patterns raises error""" + config = {"patterns": "*.py"} + with pytest.raises(ValueError, match="patterns must be a list"): + file_scanner.validate_config(config) + + async def test_invalid_max_file_size(self, file_scanner): + """Test that invalid max_file_size raises error""" + config = {"max_file_size": -1} + with pytest.raises(ValueError, match="max_file_size must be a positive integer"): + file_scanner.validate_config(config) + + async def test_invalid_max_file_size_type(self, file_scanner): + """Test that non-integer max_file_size raises error""" + config = {"max_file_size": "large"} + with pytest.raises(ValueError, match="max_file_size must be a positive integer"): + file_scanner.validate_config(config) + + +@pytest.mark.asyncio +class TestFileScannerExecution: + """Test scanner execution""" + + async def test_scan_python_files(self, file_scanner, python_test_workspace): + """Test scanning Python files""" + config = { + "patterns": ["*.py"], + "check_sensitive": False, + "calculate_hashes": False + } + + result = await file_scanner.execute(config, python_test_workspace) + + assert result.module == "file_scanner" + assert result.status == "success" + assert len(result.findings) > 0 + + # Check that Python files were found + python_files = [f for f in result.findings if f.file_path.endswith('.py')] + assert len(python_files) > 0 + + async def test_scan_all_files(self, file_scanner, python_test_workspace): + """Test scanning all files with wildcard""" + config = { + "patterns": ["*"], + "check_sensitive": False, + "calculate_hashes": False + } + + result = await file_scanner.execute(config, python_test_workspace) + + assert result.status == "success" + assert len(result.findings) > 0 + assert result.summary["total_files"] > 0 + + async def test_scan_with_multiple_patterns(self, file_scanner, python_test_workspace): + """Test scanning with multiple patterns""" + config = { + "patterns": ["*.py", "*.txt"], + "check_sensitive": False, + "calculate_hashes": False + } + + result = await file_scanner.execute(config, python_test_workspace) + + assert result.status == "success" + assert len(result.findings) > 0 + + async def test_empty_workspace(self, file_scanner, temp_workspace): + """Test scanning empty workspace""" + config = { + "patterns": ["*.py"], + "check_sensitive": False + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + assert len(result.findings) == 0 + assert result.summary["total_files"] == 0 + + +@pytest.mark.asyncio +class TestFileScannerSensitiveDetection: + """Test sensitive file detection""" + + async def test_detect_env_file(self, file_scanner, temp_workspace): + """Test detection of .env file""" + # Create .env file + (temp_workspace / ".env").write_text("API_KEY=secret123") + + config = { + "patterns": ["*"], + "check_sensitive": True, + "calculate_hashes": False + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + + # Check for sensitive file finding + sensitive_findings = [f for f in result.findings if f.category == "sensitive_file"] + assert len(sensitive_findings) > 0 + assert any(".env" in f.title for f in sensitive_findings) + + async def test_detect_private_key(self, file_scanner, temp_workspace): + """Test detection of private key file""" + # Create private key file + (temp_workspace / "id_rsa").write_text("-----BEGIN RSA PRIVATE KEY-----") + + config = { + "patterns": ["*"], + "check_sensitive": True + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + sensitive_findings = [f for f in result.findings if f.category == "sensitive_file"] + assert len(sensitive_findings) > 0 + + async def test_no_sensitive_detection_when_disabled(self, file_scanner, temp_workspace): + """Test that sensitive detection can be disabled""" + (temp_workspace / ".env").write_text("API_KEY=secret123") + + config = { + "patterns": ["*"], + "check_sensitive": False + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + sensitive_findings = [f for f in result.findings if f.category == "sensitive_file"] + assert len(sensitive_findings) == 0 + + +@pytest.mark.asyncio +class TestFileScannerHashing: + """Test file hashing functionality""" + + async def test_hash_calculation(self, file_scanner, temp_workspace): + """Test SHA256 hash calculation""" + # Create test file + test_file = temp_workspace / "test.txt" + test_file.write_text("Hello World") + + config = { + "patterns": ["*.txt"], + "calculate_hashes": True + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + + # Find the test.txt finding + txt_findings = [f for f in result.findings if "test.txt" in f.file_path] + assert len(txt_findings) > 0 + + # Check that hash was calculated + finding = txt_findings[0] + assert finding.metadata.get("file_hash") is not None + assert len(finding.metadata["file_hash"]) == 64 # SHA256 hex length + + async def test_no_hash_when_disabled(self, file_scanner, temp_workspace): + """Test that hashing can be disabled""" + test_file = temp_workspace / "test.txt" + test_file.write_text("Hello World") + + config = { + "patterns": ["*.txt"], + "calculate_hashes": False + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + txt_findings = [f for f in result.findings if "test.txt" in f.file_path] + + if len(txt_findings) > 0: + finding = txt_findings[0] + assert finding.metadata.get("file_hash") is None + + +@pytest.mark.asyncio +class TestFileScannerFileTypes: + """Test file type detection""" + + async def test_detect_python_type(self, file_scanner, temp_workspace): + """Test detection of Python file type""" + (temp_workspace / "script.py").write_text("print('hello')") + + config = {"patterns": ["*.py"]} + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + py_findings = [f for f in result.findings if "script.py" in f.file_path] + assert len(py_findings) > 0 + assert "python" in py_findings[0].metadata["file_type"] + + async def test_detect_javascript_type(self, file_scanner, temp_workspace): + """Test detection of JavaScript file type""" + (temp_workspace / "app.js").write_text("console.log('hello')") + + config = {"patterns": ["*.js"]} + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + js_findings = [f for f in result.findings if "app.js" in f.file_path] + assert len(js_findings) > 0 + assert "javascript" in js_findings[0].metadata["file_type"] + + async def test_file_type_summary(self, file_scanner, temp_workspace): + """Test that file type summary is generated""" + (temp_workspace / "script.py").write_text("print('hello')") + (temp_workspace / "app.js").write_text("console.log('hello')") + (temp_workspace / "readme.txt").write_text("Documentation") + + config = {"patterns": ["*"]} + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + assert "file_types" in result.summary + assert len(result.summary["file_types"]) > 0 + + +@pytest.mark.asyncio +class TestFileScannerSizeLimits: + """Test file size handling""" + + async def test_skip_large_files(self, file_scanner, temp_workspace): + """Test that large files are skipped""" + # Create a "large" file + large_file = temp_workspace / "large.txt" + large_file.write_text("x" * 1000) + + config = { + "patterns": ["*.txt"], + "max_file_size": 500 # Set limit smaller than file + } + + result = await file_scanner.execute(config, temp_workspace) + + # Should succeed but skip the large file + assert result.status == "success" + + # The file should still be counted but not have a detailed finding + assert result.summary["total_files"] > 0 + + async def test_process_small_files(self, file_scanner, temp_workspace): + """Test that small files are processed""" + small_file = temp_workspace / "small.txt" + small_file.write_text("small content") + + config = { + "patterns": ["*.txt"], + "max_file_size": 1048576 # 1MB + } + + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + txt_findings = [f for f in result.findings if "small.txt" in f.file_path] + assert len(txt_findings) > 0 + + +@pytest.mark.asyncio +class TestFileScannerSummary: + """Test result summary generation""" + + async def test_summary_structure(self, file_scanner, python_test_workspace): + """Test that summary has correct structure""" + config = {"patterns": ["*"]} + result = await file_scanner.execute(config, python_test_workspace) + + assert result.status == "success" + assert "total_files" in result.summary + assert "total_size_bytes" in result.summary + assert "file_types" in result.summary + assert "patterns_scanned" in result.summary + + assert isinstance(result.summary["total_files"], int) + assert isinstance(result.summary["total_size_bytes"], int) + assert isinstance(result.summary["file_types"], dict) + assert isinstance(result.summary["patterns_scanned"], list) + + async def test_summary_counts(self, file_scanner, temp_workspace): + """Test that summary counts are accurate""" + # Create known files + (temp_workspace / "file1.py").write_text("content1") + (temp_workspace / "file2.py").write_text("content2") + (temp_workspace / "file3.txt").write_text("content3") + + config = {"patterns": ["*"]} + result = await file_scanner.execute(config, temp_workspace) + + assert result.status == "success" + assert result.summary["total_files"] == 3 + assert result.summary["total_size_bytes"] > 0 diff --git a/backend/tests/unit/test_modules/test_security_analyzer.py b/backend/tests/unit/test_modules/test_security_analyzer.py new file mode 100644 index 0000000..7365a78 --- /dev/null +++ b/backend/tests/unit/test_modules/test_security_analyzer.py @@ -0,0 +1,493 @@ +""" +Unit tests for SecurityAnalyzer module +""" + +import pytest +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "toolbox")) + +from modules.analyzer.security_analyzer import SecurityAnalyzer + + +@pytest.fixture +def security_analyzer(): + """Create SecurityAnalyzer instance""" + return SecurityAnalyzer() + + +@pytest.mark.asyncio +class TestSecurityAnalyzerMetadata: + """Test SecurityAnalyzer metadata""" + + async def test_metadata_structure(self, security_analyzer): + """Test that metadata has correct structure""" + metadata = security_analyzer.get_metadata() + + assert metadata.name == "security_analyzer" + assert metadata.version == "1.0.0" + assert metadata.category == "analyzer" + assert "security" in metadata.tags + assert "vulnerabilities" in metadata.tags + assert metadata.requires_workspace is True + + +@pytest.mark.asyncio +class TestSecurityAnalyzerConfigValidation: + """Test configuration validation""" + + async def test_valid_config(self, security_analyzer): + """Test that valid config passes validation""" + config = { + "file_extensions": [".py", ".js"], + "check_secrets": True, + "check_sql": True, + "check_dangerous_functions": True + } + assert security_analyzer.validate_config(config) is True + + async def test_default_config(self, security_analyzer): + """Test that empty config uses defaults""" + config = {} + assert security_analyzer.validate_config(config) is True + + async def test_invalid_extensions_type(self, security_analyzer): + """Test that non-list extensions raises error""" + config = {"file_extensions": ".py"} + with pytest.raises(ValueError, match="file_extensions must be a list"): + security_analyzer.validate_config(config) + + +@pytest.mark.asyncio +class TestSecurityAnalyzerSecretDetection: + """Test hardcoded secret detection""" + + async def test_detect_api_key(self, security_analyzer, temp_workspace): + """Test detection of hardcoded API key""" + code_file = temp_workspace / "config.py" + code_file.write_text(""" +# Configuration file +api_key = "apikey_live_abcdefghijklmnopqrstuvwxyzabcdefghijk" +database_url = "postgresql://localhost/db" +""") + + config = { + "file_extensions": [".py"], + "check_secrets": True, + "check_sql": False, + "check_dangerous_functions": False + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + secret_findings = [f for f in result.findings if f.category == "hardcoded_secret"] + assert len(secret_findings) > 0 + assert any("API Key" in f.title for f in secret_findings) + + async def test_detect_password(self, security_analyzer, temp_workspace): + """Test detection of hardcoded password""" + code_file = temp_workspace / "auth.py" + code_file.write_text(""" +def connect(): + password = "mySecretP@ssw0rd" + return connect_db(password) +""") + + config = { + "file_extensions": [".py"], + "check_secrets": True, + "check_sql": False, + "check_dangerous_functions": False + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + secret_findings = [f for f in result.findings if f.category == "hardcoded_secret"] + assert len(secret_findings) > 0 + + async def test_detect_aws_credentials(self, security_analyzer, temp_workspace): + """Test detection of AWS credentials""" + code_file = temp_workspace / "aws_config.py" + code_file.write_text(""" +aws_access_key = "AKIAIOSFODNN7REALKEY" +aws_secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYREALKEY" +""") + + config = { + "file_extensions": [".py"], + "check_secrets": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + aws_findings = [f for f in result.findings if "AWS" in f.title] + assert len(aws_findings) >= 2 # Both access key and secret key + + async def test_no_secret_detection_when_disabled(self, security_analyzer, temp_workspace): + """Test that secret detection can be disabled""" + code_file = temp_workspace / "config.py" + code_file.write_text('api_key = "sk_live_1234567890abcdef"') + + config = { + "file_extensions": [".py"], + "check_secrets": False + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + secret_findings = [f for f in result.findings if f.category == "hardcoded_secret"] + assert len(secret_findings) == 0 + + +@pytest.mark.asyncio +class TestSecurityAnalyzerSQLInjection: + """Test SQL injection detection""" + + async def test_detect_string_concatenation(self, security_analyzer, temp_workspace): + """Test detection of SQL string concatenation""" + code_file = temp_workspace / "db.py" + code_file.write_text(""" +def get_user(user_id): + query = "SELECT * FROM users WHERE id = " + user_id + return execute(query) +""") + + config = { + "file_extensions": [".py"], + "check_secrets": False, + "check_sql": True, + "check_dangerous_functions": False + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + sql_findings = [f for f in result.findings if f.category == "sql_injection"] + assert len(sql_findings) > 0 + + async def test_detect_f_string_sql(self, security_analyzer, temp_workspace): + """Test detection of f-string in SQL""" + code_file = temp_workspace / "db.py" + code_file.write_text(""" +def get_user(name): + query = f"SELECT * FROM users WHERE name = '{name}'" + return execute(query) +""") + + config = { + "file_extensions": [".py"], + "check_sql": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + sql_findings = [f for f in result.findings if f.category == "sql_injection"] + assert len(sql_findings) > 0 + + async def test_detect_dynamic_query_building(self, security_analyzer, temp_workspace): + """Test detection of dynamic query building""" + code_file = temp_workspace / "queries.py" + code_file.write_text(""" +def search(keyword): + query = "SELECT * FROM products WHERE name LIKE " + keyword + execute(query + " ORDER BY price") +""") + + config = { + "file_extensions": [".py"], + "check_sql": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + sql_findings = [f for f in result.findings if f.category == "sql_injection"] + assert len(sql_findings) > 0 + + async def test_no_sql_detection_when_disabled(self, security_analyzer, temp_workspace): + """Test that SQL detection can be disabled""" + code_file = temp_workspace / "db.py" + code_file.write_text('query = "SELECT * FROM users WHERE id = " + user_id') + + config = { + "file_extensions": [".py"], + "check_sql": False + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + sql_findings = [f for f in result.findings if f.category == "sql_injection"] + assert len(sql_findings) == 0 + + +@pytest.mark.asyncio +class TestSecurityAnalyzerDangerousFunctions: + """Test dangerous function detection""" + + async def test_detect_eval(self, security_analyzer, temp_workspace): + """Test detection of eval() usage""" + code_file = temp_workspace / "dangerous.py" + code_file.write_text(""" +def process_input(user_input): + result = eval(user_input) + return result +""") + + config = { + "file_extensions": [".py"], + "check_secrets": False, + "check_sql": False, + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) > 0 + assert any("eval" in f.title.lower() for f in dangerous_findings) + + async def test_detect_exec(self, security_analyzer, temp_workspace): + """Test detection of exec() usage""" + code_file = temp_workspace / "runner.py" + code_file.write_text(""" +def run_code(code): + exec(code) +""") + + config = { + "file_extensions": [".py"], + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) > 0 + + async def test_detect_os_system(self, security_analyzer, temp_workspace): + """Test detection of os.system() usage""" + code_file = temp_workspace / "commands.py" + code_file.write_text(""" +import os + +def run_command(cmd): + os.system(cmd) +""") + + config = { + "file_extensions": [".py"], + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) > 0 + assert any("os.system" in f.title for f in dangerous_findings) + + async def test_detect_pickle_loads(self, security_analyzer, temp_workspace): + """Test detection of pickle.loads() usage""" + code_file = temp_workspace / "serializer.py" + code_file.write_text(""" +import pickle + +def deserialize(data): + return pickle.loads(data) +""") + + config = { + "file_extensions": [".py"], + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) > 0 + + async def test_detect_javascript_eval(self, security_analyzer, temp_workspace): + """Test detection of eval() in JavaScript""" + code_file = temp_workspace / "app.js" + code_file.write_text(""" +function processInput(userInput) { + return eval(userInput); +} +""") + + config = { + "file_extensions": [".js"], + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) > 0 + + async def test_detect_innerHTML(self, security_analyzer, temp_workspace): + """Test detection of innerHTML (XSS risk)""" + code_file = temp_workspace / "dom.js" + code_file.write_text(""" +function updateContent(html) { + document.getElementById("content").innerHTML = html; +} +""") + + config = { + "file_extensions": [".js"], + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) > 0 + + async def test_no_dangerous_detection_when_disabled(self, security_analyzer, temp_workspace): + """Test that dangerous function detection can be disabled""" + code_file = temp_workspace / "code.py" + code_file.write_text('result = eval(user_input)') + + config = { + "file_extensions": [".py"], + "check_dangerous_functions": False + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + assert len(dangerous_findings) == 0 + + +@pytest.mark.asyncio +class TestSecurityAnalyzerMultipleIssues: + """Test detection of multiple issues in same file""" + + async def test_detect_multiple_vulnerabilities(self, security_analyzer, temp_workspace): + """Test detection of multiple vulnerability types""" + code_file = temp_workspace / "vulnerable.py" + code_file.write_text(""" +import os + +# Hardcoded credentials +api_key = "apikey_live_abcdefghijklmnopqrstuvwxyzabcdef" +password = "MySecureP@ssw0rd" + +def process_query(user_input): + # SQL injection + query = "SELECT * FROM users WHERE name = " + user_input + + # Dangerous function + result = eval(user_input) + + # Command injection + os.system(user_input) + + return result +""") + + config = { + "file_extensions": [".py"], + "check_secrets": True, + "check_sql": True, + "check_dangerous_functions": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + + # Should find multiple types of issues + secret_findings = [f for f in result.findings if f.category == "hardcoded_secret"] + sql_findings = [f for f in result.findings if f.category == "sql_injection"] + dangerous_findings = [f for f in result.findings if f.category == "dangerous_function"] + + assert len(secret_findings) > 0 + assert len(sql_findings) > 0 + assert len(dangerous_findings) > 0 + + +@pytest.mark.asyncio +class TestSecurityAnalyzerSummary: + """Test result summary generation""" + + async def test_summary_structure(self, security_analyzer, temp_workspace): + """Test that summary has correct structure""" + (temp_workspace / "test.py").write_text("print('hello')") + + config = {"file_extensions": [".py"]} + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + assert "files_analyzed" in result.summary + assert "total_findings" in result.summary + assert "extensions_scanned" in result.summary + + assert isinstance(result.summary["files_analyzed"], int) + assert isinstance(result.summary["total_findings"], int) + assert isinstance(result.summary["extensions_scanned"], list) + + async def test_empty_workspace(self, security_analyzer, temp_workspace): + """Test analyzing empty workspace""" + config = {"file_extensions": [".py"]} + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "partial" # No files found + assert result.summary["files_analyzed"] == 0 + + async def test_analyze_multiple_file_types(self, security_analyzer, temp_workspace): + """Test analyzing multiple file types""" + (temp_workspace / "app.py").write_text("print('hello')") + (temp_workspace / "script.js").write_text("console.log('hello')") + (temp_workspace / "index.php").write_text("") + + config = {"file_extensions": [".py", ".js", ".php"]} + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + assert result.summary["files_analyzed"] == 3 + + +@pytest.mark.asyncio +class TestSecurityAnalyzerFalsePositives: + """Test false positive filtering""" + + async def test_skip_test_secrets(self, security_analyzer, temp_workspace): + """Test that test/example secrets are filtered""" + code_file = temp_workspace / "test_config.py" + code_file.write_text(""" +# Test configuration - should be filtered +api_key = "test_key_example" +password = "dummy_password_123" +token = "sample_token_placeholder" +""") + + config = { + "file_extensions": [".py"], + "check_secrets": True + } + + result = await security_analyzer.execute(config, temp_workspace) + + assert result.status == "success" + # These should be filtered as false positives + secret_findings = [f for f in result.findings if f.category == "hardcoded_secret"] + # Should have fewer or no findings due to false positive filtering + assert len(secret_findings) == 0 or all( + not any(fp in f.description.lower() for fp in ['test', 'example', 'dummy', 'sample']) + for f in secret_findings + ) diff --git a/backend/tests/unit/test_workflows/__init__.py b/backend/tests/unit/test_workflows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/toolbox/common/storage_activities.py b/backend/toolbox/common/storage_activities.py new file mode 100644 index 0000000..a09a83c --- /dev/null +++ b/backend/toolbox/common/storage_activities.py @@ -0,0 +1,369 @@ +""" +FuzzForge Common Storage Activities + +Activities for interacting with MinIO storage: +- get_target_activity: Download target from MinIO to local cache +- cleanup_cache_activity: Remove target from local cache +- upload_results_activity: Upload workflow results to MinIO +""" + +import logging +import os +import shutil +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError +from temporalio import activity + +# Configure logging +logger = logging.getLogger(__name__) + +# Initialize S3 client (MinIO) +s3_client = boto3.client( + 's3', + endpoint_url=os.getenv('S3_ENDPOINT', 'http://minio:9000'), + aws_access_key_id=os.getenv('S3_ACCESS_KEY', 'fuzzforge'), + aws_secret_access_key=os.getenv('S3_SECRET_KEY', 'fuzzforge123'), + region_name=os.getenv('S3_REGION', 'us-east-1'), + use_ssl=os.getenv('S3_USE_SSL', 'false').lower() == 'true' +) + +# Configuration +S3_BUCKET = os.getenv('S3_BUCKET', 'targets') +CACHE_DIR = Path(os.getenv('CACHE_DIR', '/cache')) +CACHE_MAX_SIZE_GB = int(os.getenv('CACHE_MAX_SIZE', '10').rstrip('GB')) + + +@activity.defn(name="get_target") +async def get_target_activity( + target_id: str, + run_id: str = None, + workspace_isolation: str = "isolated" +) -> str: + """ + Download target from MinIO to local cache. + + Args: + target_id: UUID of the uploaded target + run_id: Workflow run ID for isolation (required for isolated mode) + workspace_isolation: Isolation mode - "isolated" (default), "shared", or "copy-on-write" + + Returns: + Local path to the cached target workspace + + Raises: + FileNotFoundError: If target doesn't exist in MinIO + ValueError: If run_id not provided for isolated mode + Exception: For other download errors + """ + logger.info( + f"Activity: get_target (target_id={target_id}, run_id={run_id}, " + f"isolation={workspace_isolation})" + ) + + # Validate isolation mode + valid_modes = ["isolated", "shared", "copy-on-write"] + if workspace_isolation not in valid_modes: + raise ValueError( + f"Invalid workspace_isolation mode: {workspace_isolation}. " + f"Must be one of: {valid_modes}" + ) + + # Require run_id for isolated and copy-on-write modes + if workspace_isolation in ["isolated", "copy-on-write"] and not run_id: + raise ValueError( + f"run_id is required for workspace_isolation='{workspace_isolation}'" + ) + + # Define cache paths based on isolation mode + if workspace_isolation == "isolated": + # Each run gets its own isolated workspace + cache_path = CACHE_DIR / target_id / run_id + cached_file = cache_path / "target" + elif workspace_isolation == "shared": + # All runs share the same workspace (legacy behavior) + cache_path = CACHE_DIR / target_id + cached_file = cache_path / "target" + else: # copy-on-write + # Shared download, run-specific copy + shared_cache_path = CACHE_DIR / target_id / "shared" + cache_path = CACHE_DIR / target_id / run_id + cached_file = shared_cache_path / "target" + + # Handle copy-on-write mode + if workspace_isolation == "copy-on-write": + # Check if shared cache exists + if cached_file.exists(): + logger.info(f"Copy-on-write: Shared cache HIT for {target_id}") + + # Copy shared workspace to run-specific path + shared_workspace = shared_cache_path / "workspace" + run_workspace = cache_path / "workspace" + + if shared_workspace.exists(): + logger.info(f"Copying workspace to isolated run path: {run_workspace}") + cache_path.mkdir(parents=True, exist_ok=True) + shutil.copytree(shared_workspace, run_workspace) + return str(run_workspace) + else: + # Shared file exists but not extracted (non-tarball) + run_file = cache_path / "target" + cache_path.mkdir(parents=True, exist_ok=True) + shutil.copy2(cached_file, run_file) + return str(run_file) + # If shared cache doesn't exist, fall through to download + + # Check if target is already cached (isolated or shared mode) + elif cached_file.exists(): + # Update access time for LRU + cached_file.touch() + logger.info(f"Cache HIT: {target_id} (mode: {workspace_isolation})") + + # Check if workspace directory exists (extracted tarball) + workspace_dir = cache_path / "workspace" + if workspace_dir.exists() and workspace_dir.is_dir(): + logger.info(f"Returning cached workspace: {workspace_dir}") + return str(workspace_dir) + else: + # Return cached file (not a tarball) + return str(cached_file) + + # Cache miss - download from MinIO + logger.info( + f"Cache MISS: {target_id} (mode: {workspace_isolation}), " + f"downloading from MinIO..." + ) + + try: + # Create cache directory + cache_path.mkdir(parents=True, exist_ok=True) + + # Download from S3/MinIO + s3_key = f'{target_id}/target' + logger.info(f"Downloading s3://{S3_BUCKET}/{s3_key} -> {cached_file}") + + s3_client.download_file( + Bucket=S3_BUCKET, + Key=s3_key, + Filename=str(cached_file) + ) + + # Verify file was downloaded + if not cached_file.exists(): + raise FileNotFoundError(f"Downloaded file not found: {cached_file}") + + file_size = cached_file.stat().st_size + logger.info( + f"āœ“ Downloaded target {target_id} " + f"({file_size / 1024 / 1024:.2f} MB)" + ) + + # Extract tarball if it's an archive + import tarfile + workspace_dir = cache_path / "workspace" + + if tarfile.is_tarfile(str(cached_file)): + logger.info(f"Extracting tarball to {workspace_dir}...") + workspace_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(str(cached_file), 'r:*') as tar: + tar.extractall(path=workspace_dir) + + logger.info(f"āœ“ Extracted tarball to {workspace_dir}") + + # For copy-on-write mode, copy to run-specific path + if workspace_isolation == "copy-on-write": + run_cache_path = CACHE_DIR / target_id / run_id + run_workspace = run_cache_path / "workspace" + logger.info(f"Copy-on-write: Copying to {run_workspace}") + run_cache_path.mkdir(parents=True, exist_ok=True) + shutil.copytree(workspace_dir, run_workspace) + return str(run_workspace) + + return str(workspace_dir) + else: + # Not a tarball + if workspace_isolation == "copy-on-write": + # Copy file to run-specific path + run_cache_path = CACHE_DIR / target_id / run_id + run_file = run_cache_path / "target" + logger.info(f"Copy-on-write: Copying file to {run_file}") + run_cache_path.mkdir(parents=True, exist_ok=True) + shutil.copy2(cached_file, run_file) + return str(run_file) + + return str(cached_file) + + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == '404' or error_code == 'NoSuchKey': + logger.error(f"Target not found in MinIO: {target_id}") + raise FileNotFoundError(f"Target {target_id} not found in storage") + else: + logger.error(f"S3/MinIO error downloading target: {e}", exc_info=True) + raise + + except Exception as e: + logger.error(f"Failed to download target {target_id}: {e}", exc_info=True) + # Cleanup partial download + if cache_path.exists(): + shutil.rmtree(cache_path, ignore_errors=True) + raise + + +@activity.defn(name="cleanup_cache") +async def cleanup_cache_activity( + target_path: str, + workspace_isolation: str = "isolated" +) -> None: + """ + Remove target from local cache after workflow completes. + + Args: + target_path: Path to the cached target workspace (from get_target_activity) + workspace_isolation: Isolation mode used - determines cleanup scope + + Notes: + - "isolated" mode: Removes the entire run-specific directory + - "copy-on-write" mode: Removes run-specific directory, keeps shared cache + - "shared" mode: Does NOT remove cache (shared across runs) + """ + logger.info( + f"Activity: cleanup_cache (path={target_path}, " + f"isolation={workspace_isolation})" + ) + + try: + target = Path(target_path) + + # For shared mode, don't clean up (cache is shared across runs) + if workspace_isolation == "shared": + logger.info( + f"Skipping cleanup for shared workspace (mode={workspace_isolation})" + ) + return + + # For isolated and copy-on-write modes, clean up run-specific directory + # Navigate up to the run-specific directory: /cache/{target_id}/{run_id}/ + if target.name == "workspace": + # Path is .../workspace, go up one level to run directory + run_dir = target.parent + else: + # Path is a file, go up one level to run directory + run_dir = target.parent + + # Validate it's in cache and looks like a run-specific path + if run_dir.exists() and run_dir.is_relative_to(CACHE_DIR): + # Check if parent is target_id directory (validate structure) + target_id_dir = run_dir.parent + if target_id_dir.is_relative_to(CACHE_DIR): + shutil.rmtree(run_dir) + logger.info( + f"āœ“ Cleaned up run-specific directory: {run_dir} " + f"(mode={workspace_isolation})" + ) + else: + logger.warning( + f"Unexpected cache structure, skipping cleanup: {run_dir}" + ) + else: + logger.warning( + f"Cache path not in CACHE_DIR or doesn't exist: {run_dir}" + ) + + except Exception as e: + # Don't fail workflow if cleanup fails + logger.error( + f"Failed to cleanup cache {target_path}: {e}", + exc_info=True + ) + + +@activity.defn(name="upload_results") +async def upload_results_activity( + workflow_id: str, + results: dict, + results_format: str = "json" +) -> str: + """ + Upload workflow results to MinIO. + + Args: + workflow_id: Workflow execution ID + results: Results dictionary to upload + results_format: Format for results (json, sarif, etc.) + + Returns: + S3 URL to the uploaded results + """ + logger.info( + f"Activity: upload_results " + f"(workflow_id={workflow_id}, format={results_format})" + ) + + try: + import json + + # Prepare results content + if results_format == "json": + content = json.dumps(results, indent=2).encode('utf-8') + content_type = 'application/json' + file_ext = 'json' + elif results_format == "sarif": + content = json.dumps(results, indent=2).encode('utf-8') + content_type = 'application/sarif+json' + file_ext = 'sarif' + else: + # Default to JSON + content = json.dumps(results, indent=2).encode('utf-8') + content_type = 'application/json' + file_ext = 'json' + + # Upload to MinIO + s3_key = f'{workflow_id}/results.{file_ext}' + logger.info(f"Uploading results to s3://results/{s3_key}") + + s3_client.put_object( + Bucket='results', + Key=s3_key, + Body=content, + ContentType=content_type, + Metadata={ + 'workflow_id': workflow_id, + 'format': results_format + } + ) + + # Construct S3 URL + s3_endpoint = os.getenv('S3_ENDPOINT', 'http://minio:9000') + s3_url = f"{s3_endpoint}/results/{s3_key}" + + logger.info(f"āœ“ Uploaded results: {s3_url}") + return s3_url + + except Exception as e: + logger.error( + f"Failed to upload results for workflow {workflow_id}: {e}", + exc_info=True + ) + raise + + +def _check_cache_size(): + """Check total cache size and log warning if exceeding limit""" + try: + total_size = 0 + for item in CACHE_DIR.rglob('*'): + if item.is_file(): + total_size += item.stat().st_size + + total_size_gb = total_size / (1024 ** 3) + if total_size_gb > CACHE_MAX_SIZE_GB: + logger.warning( + f"Cache size ({total_size_gb:.2f} GB) exceeds " + f"limit ({CACHE_MAX_SIZE_GB} GB). Consider cleanup." + ) + + except Exception as e: + logger.error(f"Failed to check cache size: {e}") diff --git a/backend/toolbox/modules/analyzer/llm_analyzer.py b/backend/toolbox/modules/analyzer/llm_analyzer.py new file mode 100644 index 0000000..b3b1374 --- /dev/null +++ b/backend/toolbox/modules/analyzer/llm_analyzer.py @@ -0,0 +1,349 @@ +""" +LLM Analyzer Module - Uses AI to analyze code for security issues +""" + +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +import logging +from pathlib import Path +from typing import Dict, Any, List + +try: + from toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult +except ImportError: + try: + from modules.base import BaseModule, ModuleMetadata, ModuleResult + except ImportError: + from src.toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult + +logger = logging.getLogger(__name__) + + +class LLMAnalyzer(BaseModule): + """ + Uses an LLM to analyze code for potential security issues. + + This module: + - Sends code to an LLM agent via A2A protocol + - Asks the LLM to identify security vulnerabilities + - Collects findings and returns them in structured format + """ + + def get_metadata(self) -> ModuleMetadata: + """Get module metadata""" + return ModuleMetadata( + name="llm_analyzer", + version="1.0.0", + description="Uses AI to analyze code for security issues", + author="FuzzForge Team", + category="analyzer", + tags=["llm", "ai", "security", "analysis"], + input_schema={ + "agent_url": { + "type": "string", + "description": "A2A agent endpoint URL", + "default": "http://fuzzforge-task-agent:8000/a2a/litellm_agent" + }, + "llm_model": { + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o-mini" + }, + "llm_provider": { + "type": "string", + "description": "LLM provider (openai, anthropic, etc.)", + "default": "openai" + }, + "file_patterns": { + "type": "array", + "items": {"type": "string"}, + "description": "File patterns to analyze", + "default": ["*.py", "*.js", "*.ts", "*.java", "*.go"] + }, + "max_files": { + "type": "integer", + "description": "Maximum number of files to analyze", + "default": 5 + }, + "max_file_size": { + "type": "integer", + "description": "Maximum file size in bytes", + "default": 50000 # 50KB + }, + "timeout": { + "type": "integer", + "description": "Timeout per file in seconds", + "default": 60 + } + }, + output_schema={ + "findings": { + "type": "array", + "description": "Security issues identified by LLM" + } + }, + requires_workspace=True + ) + + def validate_config(self, config: Dict[str, Any]) -> bool: + """Validate module configuration""" + # Lazy import to avoid Temporal sandbox restrictions + try: + from fuzzforge_ai.a2a_wrapper import send_agent_task # noqa: F401 + except ImportError: + raise RuntimeError( + "A2A wrapper not available. Ensure fuzzforge_ai module is accessible." + ) + + agent_url = config.get("agent_url") + if not agent_url or not isinstance(agent_url, str): + raise ValueError("agent_url must be a valid URL string") + + max_files = config.get("max_files", 5) + if not isinstance(max_files, int) or max_files <= 0: + raise ValueError("max_files must be a positive integer") + + return True + + async def execute(self, config: Dict[str, Any], workspace: Path) -> ModuleResult: + """ + Execute the LLM analysis module. + + Args: + config: Module configuration + workspace: Path to the workspace containing code to analyze + + Returns: + ModuleResult with findings from LLM analysis + """ + # Start execution timer + self.start_timer() + + logger.info(f"Starting LLM analysis in workspace: {workspace}") + + # Extract configuration + agent_url = config.get("agent_url", "http://fuzzforge-task-agent:8000/a2a/litellm_agent") + llm_model = config.get("llm_model", "gpt-4o-mini") + llm_provider = config.get("llm_provider", "openai") + file_patterns = config.get("file_patterns", ["*.py", "*.js", "*.ts", "*.java", "*.go"]) + max_files = config.get("max_files", 5) + max_file_size = config.get("max_file_size", 50000) + timeout = config.get("timeout", 60) + + # Find files to analyze + files_to_analyze = [] + for pattern in file_patterns: + for file_path in workspace.rglob(pattern): + if file_path.is_file(): + try: + # Check file size + if file_path.stat().st_size > max_file_size: + logger.debug(f"Skipping {file_path} (too large)") + continue + + files_to_analyze.append(file_path) + + if len(files_to_analyze) >= max_files: + break + except Exception as e: + logger.warning(f"Error checking file {file_path}: {e}") + continue + + if len(files_to_analyze) >= max_files: + break + + logger.info(f"Found {len(files_to_analyze)} files to analyze") + + # Analyze each file + all_findings = [] + for file_path in files_to_analyze: + logger.info(f"Analyzing: {file_path.relative_to(workspace)}") + + try: + findings = await self._analyze_file( + file_path=file_path, + workspace=workspace, + agent_url=agent_url, + llm_model=llm_model, + llm_provider=llm_provider, + timeout=timeout + ) + all_findings.extend(findings) + + except Exception as e: + logger.error(f"Error analyzing {file_path}: {e}") + # Continue with next file + continue + + logger.info(f"LLM analysis complete. Found {len(all_findings)} issues.") + + # Create result using base module helper + return self.create_result( + findings=all_findings, + status="success", + summary={ + "files_analyzed": len(files_to_analyze), + "total_findings": len(all_findings), + "agent_url": agent_url, + "model": f"{llm_provider}/{llm_model}" + } + ) + + async def _analyze_file( + self, + file_path: Path, + workspace: Path, + agent_url: str, + llm_model: str, + llm_provider: str, + timeout: int + ) -> List[Dict[str, Any]]: + """Analyze a single file with LLM""" + + # Read file content + try: + with open(file_path, 'r', encoding='utf-8') as f: + code_content = f.read() + except Exception as e: + logger.error(f"Failed to read {file_path}: {e}") + return [] + + # Determine language from extension + extension = file_path.suffix.lower() + language_map = { + ".py": "python", + ".js": "javascript", + ".ts": "typescript", + ".java": "java", + ".go": "go", + ".rs": "rust", + ".c": "c", + ".cpp": "cpp", + } + language = language_map.get(extension, "code") + + # Build prompt for LLM + system_prompt = ( + "You are a security code analyzer. Analyze the provided code and identify " + "potential security vulnerabilities, bugs, and code quality issues. " + "For each issue found, respond in this exact format:\n" + "ISSUE: [short title]\n" + "SEVERITY: [error/warning/note]\n" + "LINE: [line number or 'unknown']\n" + "DESCRIPTION: [detailed explanation]\n\n" + "If no issues are found, respond with 'NO_ISSUES_FOUND'." + ) + + user_message = ( + f"Analyze this {language} code for security vulnerabilities:\n\n" + f"File: {file_path.relative_to(workspace)}\n\n" + f"```{language}\n{code_content}\n```" + ) + + # Call LLM via A2A wrapper (lazy import to avoid Temporal sandbox restrictions) + try: + from fuzzforge_ai.a2a_wrapper import send_agent_task + + result = await send_agent_task( + url=agent_url, + model=llm_model, + provider=llm_provider, + prompt=system_prompt, + message=user_message, + context=f"llm_analysis_{file_path.stem}", + timeout=float(timeout) + ) + + llm_response = result.text + + except Exception as e: + logger.error(f"A2A call failed for {file_path}: {e}") + return [] + + # Parse LLM response into findings + findings = self._parse_llm_response( + llm_response=llm_response, + file_path=file_path, + workspace=workspace + ) + + return findings + + def _parse_llm_response( + self, + llm_response: str, + file_path: Path, + workspace: Path + ) -> List: + """Parse LLM response into structured findings""" + + if "NO_ISSUES_FOUND" in llm_response: + return [] + + findings = [] + relative_path = str(file_path.relative_to(workspace)) + + # Simple parser for the expected format + lines = llm_response.split('\n') + current_issue = {} + + for line in lines: + line = line.strip() + + if line.startswith("ISSUE:"): + # Save previous issue if exists + if current_issue: + findings.append(self._create_module_finding(current_issue, relative_path)) + current_issue = {"title": line.replace("ISSUE:", "").strip()} + + elif line.startswith("SEVERITY:"): + current_issue["severity"] = line.replace("SEVERITY:", "").strip().lower() + + elif line.startswith("LINE:"): + line_num = line.replace("LINE:", "").strip() + try: + current_issue["line"] = int(line_num) + except ValueError: + current_issue["line"] = None + + elif line.startswith("DESCRIPTION:"): + current_issue["description"] = line.replace("DESCRIPTION:", "").strip() + + # Save last issue + if current_issue: + findings.append(self._create_module_finding(current_issue, relative_path)) + + return findings + + def _create_module_finding(self, issue: Dict[str, Any], file_path: str): + """Create a ModuleFinding from parsed issue""" + + severity_map = { + "error": "critical", + "warning": "medium", + "note": "low", + "info": "low" + } + + # Use base class helper to create proper ModuleFinding + return self.create_finding( + title=issue.get("title", "Security issue detected"), + description=issue.get("description", ""), + severity=severity_map.get(issue.get("severity", "warning"), "medium"), + category="security", + file_path=file_path, + line_start=issue.get("line"), + metadata={ + "tool": "llm-analyzer", + "type": "llm-security-analysis" + } + ) diff --git a/backend/toolbox/modules/analyzer/security_analyzer.py b/backend/toolbox/modules/analyzer/security_analyzer.py index 8688c18..3b4a2ea 100644 --- a/backend/toolbox/modules/analyzer/security_analyzer.py +++ b/backend/toolbox/modules/analyzer/security_analyzer.py @@ -16,7 +16,7 @@ Security Analyzer Module - Analyzes code for security vulnerabilities import logging import re from pathlib import Path -from typing import Dict, Any, List, Optional +from typing import Dict, Any, List try: from toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding diff --git a/backend/toolbox/modules/base.py b/backend/toolbox/modules/base.py index 62a722c..dcef98d 100644 --- a/backend/toolbox/modules/base.py +++ b/backend/toolbox/modules/base.py @@ -17,7 +17,6 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Dict, Any, List, Optional from pydantic import BaseModel, Field -from datetime import datetime import logging logger = logging.getLogger(__name__) diff --git a/backend/toolbox/modules/fuzzer/__init__.py b/backend/toolbox/modules/fuzzer/__init__.py new file mode 100644 index 0000000..ad0d1ba --- /dev/null +++ b/backend/toolbox/modules/fuzzer/__init__.py @@ -0,0 +1,10 @@ +""" +Fuzzing modules for FuzzForge + +This package contains fuzzing modules for different fuzzing engines. +""" + +from .atheris_fuzzer import AtherisFuzzer +from .cargo_fuzzer import CargoFuzzer + +__all__ = ["AtherisFuzzer", "CargoFuzzer"] diff --git a/backend/toolbox/modules/fuzzer/atheris_fuzzer.py b/backend/toolbox/modules/fuzzer/atheris_fuzzer.py new file mode 100644 index 0000000..3f0c42d --- /dev/null +++ b/backend/toolbox/modules/fuzzer/atheris_fuzzer.py @@ -0,0 +1,608 @@ +""" +Atheris Fuzzer Module + +Reusable module for fuzzing Python code using Atheris. +Discovers and fuzzes user-provided Python targets with TestOneInput() function. +""" + +import asyncio +import base64 +import importlib.util +import logging +import multiprocessing +import os +import sys +import time +from datetime import datetime +from pathlib import Path +from typing import Dict, Any, List, Optional, Callable +import uuid + +import httpx +from modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding + +logger = logging.getLogger(__name__) + + +def _run_atheris_in_subprocess( + target_path_str: str, + corpus_dir_str: str, + max_iterations: int, + timeout_seconds: int, + shared_crashes: Any, + exec_counter: multiprocessing.Value, + crash_counter: multiprocessing.Value, + coverage_counter: multiprocessing.Value +): + """ + Run atheris.Fuzz() in a separate process to isolate os._exit() calls. + + This function runs in a subprocess and loads the target module, + sets up atheris, and runs fuzzing. Stats are communicated via shared memory. + + Args: + target_path_str: String path to target file + corpus_dir_str: String path to corpus directory + max_iterations: Maximum fuzzing iterations + timeout_seconds: Timeout in seconds + shared_crashes: Manager().list() for storing crash details + exec_counter: Shared counter for executions + crash_counter: Shared counter for crashes + coverage_counter: Shared counter for coverage edges + """ + import atheris + import importlib.util + import traceback + from pathlib import Path + + target_path = Path(target_path_str) + total_executions = 0 + + # NOTE: Crash details are written directly to shared_crashes (Manager().list()) + # so they can be accessed by parent process after subprocess exits. + # We don't use a local crashes list because os._exit() prevents cleanup code. + + try: + # Load target module in subprocess + module_name = f"fuzz_target_{uuid.uuid4().hex[:8]}" + spec = importlib.util.spec_from_file_location(module_name, target_path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load module from {target_path}") + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + + if not hasattr(module, "TestOneInput"): + raise AttributeError("Module does not have TestOneInput() function") + + test_one_input = module.TestOneInput + + # Wrapper to track executions and crashes + def fuzz_wrapper(data): + nonlocal total_executions + total_executions += 1 + + # Update shared counter for live stats + with exec_counter.get_lock(): + exec_counter.value += 1 + + try: + test_one_input(data) + except Exception as e: + # Capture crash details to shared memory + crash_info = { + "input": bytes(data), # Convert to bytes for serialization + "exception_type": type(e).__name__, + "exception_message": str(e), + "stack_trace": traceback.format_exc(), + "execution": total_executions + } + # Write to shared memory so parent process can access crash details + shared_crashes.append(crash_info) + + # Update shared crash counter + with crash_counter.get_lock(): + crash_counter.value += 1 + + # Re-raise so Atheris detects it + raise + + # Check for dictionary file in target directory + dict_args = [] + target_dir = target_path.parent + for dict_name in ["fuzz.dict", "fuzzing.dict", "dict.txt"]: + dict_path = target_dir / dict_name + if dict_path.exists(): + dict_args.append(f"-dict={dict_path}") + break + + # Configure Atheris + atheris_args = [ + "atheris_fuzzer", + f"-runs={max_iterations}", + f"-max_total_time={timeout_seconds}", + "-print_final_stats=1" + ] + dict_args + [corpus_dir_str] # Corpus directory as positional arg + + atheris.Setup(atheris_args, fuzz_wrapper) + + # Run fuzzing (this will call os._exit() when done) + atheris.Fuzz() + + except SystemExit: + # Atheris exits when done - this is normal + # Crash details already written to shared_crashes + pass + except Exception: + # Fatal error - traceback already written to shared memory + # via crash handler in fuzz_wrapper + pass + + +class AtherisFuzzer(BaseModule): + """ + Atheris fuzzing module - discovers and fuzzes Python code. + + This module can be used by any workflow to fuzz Python targets. + """ + + def __init__(self): + super().__init__() + self.crashes = [] + self.total_executions = 0 + self.start_time = None + self.last_stats_time = 0 + self.run_id = None + + def get_metadata(self) -> ModuleMetadata: + """Return module metadata""" + return ModuleMetadata( + name="atheris_fuzzer", + version="1.0.0", + description="Python fuzzing using Atheris - discovers and fuzzes TestOneInput() functions", + author="FuzzForge Team", + category="fuzzer", + tags=["fuzzing", "atheris", "python", "coverage"], + input_schema={ + "type": "object", + "properties": { + "target_file": { + "type": "string", + "description": "Python file with TestOneInput() function (auto-discovered if not specified)" + }, + "max_iterations": { + "type": "integer", + "description": "Maximum fuzzing iterations", + "default": 100000 + }, + "timeout_seconds": { + "type": "integer", + "description": "Fuzzing timeout in seconds", + "default": 300 + }, + "stats_callback": { + "description": "Optional callback for real-time statistics" + } + } + }, + requires_workspace=True + ) + + def validate_config(self, config: Dict[str, Any]) -> bool: + """Validate fuzzing configuration""" + max_iterations = config.get("max_iterations", 100000) + if not isinstance(max_iterations, int) or max_iterations <= 0: + raise ValueError(f"max_iterations must be positive integer, got: {max_iterations}") + + timeout = config.get("timeout_seconds", 300) + if not isinstance(timeout, int) or timeout <= 0: + raise ValueError(f"timeout_seconds must be positive integer, got: {timeout}") + + return True + + async def execute(self, config: Dict[str, Any], workspace: Path) -> ModuleResult: + """ + Execute Atheris fuzzing on user code. + + Args: + config: Fuzzing configuration + workspace: Path to user's uploaded code + + Returns: + ModuleResult with crash findings + """ + self.start_timer() + self.start_time = time.time() + + # Validate configuration + self.validate_config(config) + self.validate_workspace(workspace) + + # Extract config + target_file = config.get("target_file") + max_iterations = config.get("max_iterations", 100000) + timeout_seconds = config.get("timeout_seconds", 300) + stats_callback = config.get("stats_callback") + self.run_id = config.get("run_id") + + logger.info( + f"Starting Atheris fuzzing (max_iterations={max_iterations}, " + f"timeout={timeout_seconds}s, target={target_file or 'auto-discover'})" + ) + + try: + # Step 1: Discover or load target + target_path = self._discover_target(workspace, target_file) + logger.info(f"Using fuzz target: {target_path}") + + # Step 2: Load target module + test_one_input = self._load_target_module(target_path) + logger.info(f"Loaded TestOneInput function from {target_path}") + + # Step 3: Run fuzzing + await self._run_fuzzing( + test_one_input=test_one_input, + target_path=target_path, + workspace=workspace, + max_iterations=max_iterations, + timeout_seconds=timeout_seconds, + stats_callback=stats_callback + ) + + # Step 4: Generate findings from crashes + findings = await self._generate_findings(target_path) + + logger.info( + f"Fuzzing completed: {self.total_executions} executions, " + f"{len(self.crashes)} crashes found" + ) + + # Generate SARIF report (always, even with no findings) + from modules.reporter import SARIFReporter + reporter = SARIFReporter() + reporter_config = { + "findings": findings, + "tool_name": "Atheris Fuzzer", + "tool_version": self._metadata.version + } + reporter_result = await reporter.execute(reporter_config, workspace) + sarif_report = reporter_result.sarif + + return ModuleResult( + module=self._metadata.name, + version=self._metadata.version, + status="success", + execution_time=self.get_execution_time(), + findings=findings, + summary={ + "total_executions": self.total_executions, + "crashes_found": len(self.crashes), + "execution_time": self.get_execution_time(), + "target_file": str(target_path.relative_to(workspace)) + }, + metadata={ + "max_iterations": max_iterations, + "timeout_seconds": timeout_seconds + }, + sarif=sarif_report + ) + + except Exception as e: + logger.error(f"Fuzzing failed: {e}", exc_info=True) + return self.create_result( + findings=[], + status="failed", + error=str(e) + ) + + def _discover_target(self, workspace: Path, target_file: Optional[str]) -> Path: + """ + Discover fuzz target in workspace. + + Args: + workspace: Path to workspace + target_file: Explicit target file or None for auto-discovery + + Returns: + Path to target file + """ + if target_file: + # Use specified target + target_path = workspace / target_file + if not target_path.exists(): + raise FileNotFoundError(f"Target file not found: {target_file}") + return target_path + + # Auto-discover: look for fuzz_*.py or *_fuzz.py + logger.info("Auto-discovering fuzz targets...") + + candidates = [] + # Use rglob for recursive search (searches all subdirectories) + for pattern in ["fuzz_*.py", "*_fuzz.py", "fuzz_target.py"]: + matches = list(workspace.rglob(pattern)) + candidates.extend(matches) + + if not candidates: + raise FileNotFoundError( + "No fuzz targets found. Expected files matching: fuzz_*.py, *_fuzz.py, or fuzz_target.py" + ) + + # Use first candidate + target = candidates[0] + if len(candidates) > 1: + logger.warning( + f"Multiple fuzz targets found: {[str(c) for c in candidates]}. " + f"Using: {target.name}" + ) + + return target + + def _load_target_module(self, target_path: Path) -> Callable: + """ + Load target module and get TestOneInput function. + + Args: + target_path: Path to Python file with TestOneInput + + Returns: + TestOneInput function + """ + # Add target directory to sys.path + target_dir = target_path.parent + if str(target_dir) not in sys.path: + sys.path.insert(0, str(target_dir)) + + # Load module dynamically + module_name = target_path.stem + spec = importlib.util.spec_from_file_location(module_name, target_path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load module from {target_path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Get TestOneInput function + if not hasattr(module, "TestOneInput"): + raise AttributeError( + f"Module {module_name} does not have TestOneInput() function. " + "Atheris requires a TestOneInput(data: bytes) function." + ) + + return module.TestOneInput + + async def _run_fuzzing( + self, + test_one_input: Callable, + target_path: Path, + workspace: Path, + max_iterations: int, + timeout_seconds: int, + stats_callback: Optional[Callable] = None + ): + """ + Run Atheris fuzzing with real-time monitoring. + + Args: + test_one_input: TestOneInput function to fuzz (not used, loaded in subprocess) + target_path: Path to target file + workspace: Path to workspace directory + max_iterations: Max iterations + timeout_seconds: Timeout in seconds + stats_callback: Optional callback for stats + """ + self.crashes = [] + self.total_executions = 0 + + # Create corpus directory in workspace + corpus_dir = workspace / ".fuzzforge_corpus" + corpus_dir.mkdir(exist_ok=True) + logger.info(f"Using corpus directory: {corpus_dir}") + + logger.info(f"Starting Atheris fuzzer in subprocess (max_runs={max_iterations}, timeout={timeout_seconds}s)...") + + # Create shared memory for subprocess communication + ctx = multiprocessing.get_context('spawn') + manager = ctx.Manager() + shared_crashes = manager.list() # Shared list for crash details + exec_counter = ctx.Value('i', 0) # Shared execution counter + crash_counter = ctx.Value('i', 0) # Shared crash counter + coverage_counter = ctx.Value('i', 0) # Shared coverage counter + + # Start fuzzing in subprocess + process = ctx.Process( + target=_run_atheris_in_subprocess, + args=(str(target_path), str(corpus_dir), max_iterations, timeout_seconds, shared_crashes, exec_counter, crash_counter, coverage_counter) + ) + + # Run fuzzing in a separate task with monitoring + async def monitor_stats(): + """Monitor and report stats every 0.5 seconds""" + while True: + await asyncio.sleep(0.5) + + if stats_callback: + elapsed = time.time() - self.start_time + # Read from shared counters + current_execs = exec_counter.value + current_crashes = crash_counter.value + current_coverage = coverage_counter.value + execs_per_sec = current_execs / elapsed if elapsed > 0 else 0 + + # Count corpus files + try: + corpus_size = len(list(corpus_dir.iterdir())) if corpus_dir.exists() else 0 + except Exception: + corpus_size = 0 + + # TODO: Get real coverage from Atheris + # For now use corpus_size as proxy + coverage_value = current_coverage if current_coverage > 0 else corpus_size + + await stats_callback({ + "total_execs": current_execs, + "execs_per_sec": execs_per_sec, + "crashes": current_crashes, + "corpus_size": corpus_size, + "coverage": coverage_value, # Using corpus as coverage proxy + "elapsed_time": int(elapsed) + }) + + # Start monitoring task + monitor_task = None + if stats_callback: + monitor_task = asyncio.create_task(monitor_stats()) + + try: + # Start subprocess + process.start() + logger.info(f"Fuzzing subprocess started (PID: {process.pid})") + + # Wait for subprocess to complete + while process.is_alive(): + await asyncio.sleep(0.1) + + # NOTE: We cannot use result_queue because Atheris calls os._exit() + # which terminates immediately without putting results in the queue. + # Instead, we rely on shared memory (Manager().list() and Value counters). + + # Read final values from shared memory + self.total_executions = exec_counter.value + total_crashes = crash_counter.value + + # Read crash details from shared memory and convert to our format + self.crashes = [] + for crash_data in shared_crashes: + # Reconstruct crash info with exception object + crash_info = { + "input": crash_data["input"], + "exception": Exception(crash_data["exception_message"]), + "exception_type": crash_data["exception_type"], + "stack_trace": crash_data["stack_trace"], + "execution": crash_data["execution"] + } + self.crashes.append(crash_info) + + logger.warning( + f"Crash found (execution {crash_data['execution']}): " + f"{crash_data['exception_type']}: {crash_data['exception_message']}" + ) + + logger.info(f"Fuzzing completed: {self.total_executions} executions, {total_crashes} crashes found") + + # Send final stats update + if stats_callback: + elapsed = time.time() - self.start_time + execs_per_sec = self.total_executions / elapsed if elapsed > 0 else 0 + + # Count final corpus size + try: + final_corpus_size = len(list(corpus_dir.iterdir())) if corpus_dir.exists() else 0 + except Exception: + final_corpus_size = 0 + + # TODO: Parse coverage from Atheris output + # For now, use corpus size as proxy (corpus grows with coverage) + # libFuzzer writes coverage to stdout but sys.stdout redirection + # doesn't work because it writes to FD 1 directly from C++ + final_coverage = coverage_counter.value if coverage_counter.value > 0 else final_corpus_size + + await stats_callback({ + "total_execs": self.total_executions, + "execs_per_sec": execs_per_sec, + "crashes": total_crashes, + "corpus_size": final_corpus_size, + "coverage": final_coverage, + "elapsed_time": int(elapsed) + }) + + # Wait for process to fully terminate + process.join(timeout=5) + + if process.exitcode is not None and process.exitcode != 0: + logger.warning(f"Subprocess exited with code: {process.exitcode}") + + except Exception as e: + logger.error(f"Fuzzing execution error: {e}") + if process.is_alive(): + logger.warning("Terminating fuzzing subprocess...") + process.terminate() + process.join(timeout=5) + if process.is_alive(): + process.kill() + raise + finally: + # Stop monitoring + if monitor_task: + monitor_task.cancel() + try: + await monitor_task + except asyncio.CancelledError: + pass + + async def _generate_findings(self, target_path: Path) -> List[ModuleFinding]: + """ + Generate ModuleFinding objects from crashes. + + Args: + target_path: Path to target file + + Returns: + List of findings + """ + findings = [] + + for idx, crash in enumerate(self.crashes): + # Encode crash input for storage + crash_input_b64 = base64.b64encode(crash["input"]).decode() + + finding = self.create_finding( + title=f"Crash: {crash['exception_type']}", + description=( + f"Atheris found crash during fuzzing:\n" + f"Exception: {crash['exception_type']}\n" + f"Message: {str(crash['exception'])}\n" + f"Execution: {crash['execution']}" + ), + severity="critical", + category="crash", + file_path=str(target_path), + metadata={ + "crash_input_base64": crash_input_b64, + "crash_input_hex": crash["input"].hex(), + "exception_type": crash["exception_type"], + "stack_trace": crash["stack_trace"], + "execution_number": crash["execution"] + }, + recommendation=( + "Review the crash stack trace and input to identify the vulnerability. " + "The crash input is provided in base64 and hex formats for reproduction." + ) + ) + findings.append(finding) + + # Report crash to backend for real-time monitoring + if self.run_id: + try: + crash_report = { + "run_id": self.run_id, + "crash_id": f"crash_{idx + 1}", + "timestamp": datetime.utcnow().isoformat(), + "crash_type": crash["exception_type"], + "stack_trace": crash["stack_trace"], + "input_file": crash_input_b64, + "severity": "critical", + "exploitability": "unknown" + } + + backend_url = os.getenv("BACKEND_URL", "http://backend:8000") + async with httpx.AsyncClient(timeout=5.0) as client: + await client.post( + f"{backend_url}/fuzzing/{self.run_id}/crash", + json=crash_report + ) + logger.debug(f"Crash report sent to backend: {crash_report['crash_id']}") + except Exception as e: + logger.debug(f"Failed to post crash report to backend: {e}") + + return findings diff --git a/backend/toolbox/modules/fuzzer/cargo_fuzzer.py b/backend/toolbox/modules/fuzzer/cargo_fuzzer.py new file mode 100644 index 0000000..c4fc746 --- /dev/null +++ b/backend/toolbox/modules/fuzzer/cargo_fuzzer.py @@ -0,0 +1,455 @@ +""" +Cargo Fuzzer Module + +Reusable module for fuzzing Rust code using cargo-fuzz (libFuzzer). +Discovers and fuzzes user-provided Rust targets with fuzz_target!() macros. +""" + +import asyncio +import logging +import os +import re +import time +from pathlib import Path +from typing import Dict, Any, List, Optional, Callable + +from modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding + +logger = logging.getLogger(__name__) + + +class CargoFuzzer(BaseModule): + """ + Cargo-fuzz (libFuzzer) fuzzer module for Rust code. + + Discovers fuzz targets in user's Rust project and runs cargo-fuzz + to find crashes, undefined behavior, and memory safety issues. + """ + + def get_metadata(self) -> ModuleMetadata: + """Get module metadata""" + return ModuleMetadata( + name="cargo_fuzz", + version="0.11.2", + description="Fuzz Rust code using cargo-fuzz with libFuzzer backend", + author="FuzzForge Team", + category="fuzzer", + tags=["fuzzing", "rust", "cargo-fuzz", "libfuzzer", "memory-safety"], + input_schema={ + "type": "object", + "properties": { + "target_name": { + "type": "string", + "description": "Fuzz target name (auto-discovered if not specified)" + }, + "max_iterations": { + "type": "integer", + "default": 1000000, + "description": "Maximum fuzzing iterations" + }, + "timeout_seconds": { + "type": "integer", + "default": 1800, + "description": "Fuzzing timeout in seconds" + }, + "sanitizer": { + "type": "string", + "enum": ["address", "memory", "undefined"], + "default": "address", + "description": "Sanitizer to use (address, memory, undefined)" + } + } + }, + output_schema={ + "type": "object", + "properties": { + "findings": { + "type": "array", + "description": "Crashes and memory safety issues found" + }, + "summary": { + "type": "object", + "description": "Fuzzing execution summary" + } + } + } + ) + + def validate_config(self, config: Dict[str, Any]) -> bool: + """Validate configuration""" + max_iterations = config.get("max_iterations", 1000000) + if not isinstance(max_iterations, int) or max_iterations < 1: + raise ValueError("max_iterations must be a positive integer") + + timeout = config.get("timeout_seconds", 1800) + if not isinstance(timeout, int) or timeout < 1: + raise ValueError("timeout_seconds must be a positive integer") + + sanitizer = config.get("sanitizer", "address") + if sanitizer not in ["address", "memory", "undefined"]: + raise ValueError("sanitizer must be one of: address, memory, undefined") + + return True + + async def execute( + self, + config: Dict[str, Any], + workspace: Path, + stats_callback: Optional[Callable] = None + ) -> ModuleResult: + """ + Execute cargo-fuzz on user's Rust code. + + Args: + config: Fuzzer configuration + workspace: Path to workspace directory containing Rust project + stats_callback: Optional callback for real-time stats updates + + Returns: + ModuleResult containing findings and summary + """ + self.start_timer() + + try: + # Validate inputs + self.validate_config(config) + self.validate_workspace(workspace) + + logger.info(f"Running cargo-fuzz on {workspace}") + + # Step 1: Discover fuzz targets + targets = await self._discover_fuzz_targets(workspace) + if not targets: + return self.create_result( + findings=[], + status="failed", + error="No fuzz targets found. Expected fuzz targets in fuzz/fuzz_targets/" + ) + + # Get target name from config or use first discovered target + target_name = config.get("target_name") + if not target_name: + target_name = targets[0] + logger.info(f"No target specified, using first discovered target: {target_name}") + elif target_name not in targets: + return self.create_result( + findings=[], + status="failed", + error=f"Target '{target_name}' not found. Available targets: {', '.join(targets)}" + ) + + # Step 2: Build fuzz target + logger.info(f"Building fuzz target: {target_name}") + build_success = await self._build_fuzz_target(workspace, target_name, config) + if not build_success: + return self.create_result( + findings=[], + status="failed", + error=f"Failed to build fuzz target: {target_name}" + ) + + # Step 3: Run fuzzing + logger.info(f"Starting fuzzing: {target_name}") + findings, stats = await self._run_fuzzing( + workspace, + target_name, + config, + stats_callback + ) + + # Step 4: Parse crash artifacts + crash_findings = await self._parse_crash_artifacts(workspace, target_name) + findings.extend(crash_findings) + + logger.info(f"Fuzzing completed: {len(findings)} crashes found") + + return self.create_result( + findings=findings, + status="success", + summary=stats + ) + + except Exception as e: + logger.error(f"Cargo fuzzer failed: {e}") + return self.create_result( + findings=[], + status="failed", + error=str(e) + ) + + async def _discover_fuzz_targets(self, workspace: Path) -> List[str]: + """ + Discover fuzz targets in the project. + + Looks for fuzz targets in fuzz/fuzz_targets/ directory. + """ + fuzz_targets_dir = workspace / "fuzz" / "fuzz_targets" + if not fuzz_targets_dir.exists(): + logger.warning(f"No fuzz targets directory found: {fuzz_targets_dir}") + return [] + + targets = [] + for file in fuzz_targets_dir.glob("*.rs"): + target_name = file.stem + targets.append(target_name) + logger.info(f"Discovered fuzz target: {target_name}") + + return targets + + async def _build_fuzz_target( + self, + workspace: Path, + target_name: str, + config: Dict[str, Any] + ) -> bool: + """Build the fuzz target with instrumentation""" + try: + sanitizer = config.get("sanitizer", "address") + + # Build command + cmd = [ + "cargo", "fuzz", "build", + target_name, + f"--sanitizer={sanitizer}" + ] + + logger.debug(f"Build command: {' '.join(cmd)}") + + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=workspace, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + + stdout, stderr = await proc.communicate() + + if proc.returncode != 0: + logger.error(f"Build failed: {stderr.decode()}") + return False + + logger.info("Build successful") + return True + + except Exception as e: + logger.error(f"Build error: {e}") + return False + + async def _run_fuzzing( + self, + workspace: Path, + target_name: str, + config: Dict[str, Any], + stats_callback: Optional[Callable] + ) -> tuple[List[ModuleFinding], Dict[str, Any]]: + """ + Run cargo-fuzz and collect statistics. + + Returns: + Tuple of (findings, stats_dict) + """ + max_iterations = config.get("max_iterations", 1000000) + timeout_seconds = config.get("timeout_seconds", 1800) + sanitizer = config.get("sanitizer", "address") + + findings = [] + stats = { + "total_executions": 0, + "crashes_found": 0, + "corpus_size": 0, + "coverage": 0.0, + "execution_time": 0.0 + } + + try: + # Cargo fuzz run command + cmd = [ + "cargo", "fuzz", "run", + target_name, + f"--sanitizer={sanitizer}", + "--", + f"-runs={max_iterations}", + f"-max_total_time={timeout_seconds}" + ] + + logger.debug(f"Fuzz command: {' '.join(cmd)}") + + start_time = time.time() + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=workspace, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT + ) + + # Monitor output and extract stats + last_stats_time = time.time() + async for line in proc.stdout: + line_str = line.decode('utf-8', errors='ignore').strip() + + # Parse libFuzzer stats + # Example: "#12345 NEW cov: 123 ft: 456 corp: 10/234b" + stats_match = re.match(r'#(\d+)\s+.*cov:\s*(\d+).*corp:\s*(\d+)', line_str) + if stats_match: + execs = int(stats_match.group(1)) + cov = int(stats_match.group(2)) + corp = int(stats_match.group(3)) + + stats["total_executions"] = execs + stats["coverage"] = float(cov) + stats["corpus_size"] = corp + stats["execution_time"] = time.time() - start_time + + # Invoke stats callback for real-time monitoring + if stats_callback and time.time() - last_stats_time >= 0.5: + await stats_callback({ + "total_execs": execs, + "execs_per_sec": execs / stats["execution_time"] if stats["execution_time"] > 0 else 0, + "crashes": stats["crashes_found"], + "coverage": cov, + "corpus_size": corp, + "elapsed_time": int(stats["execution_time"]) + }) + last_stats_time = time.time() + + # Detect crash line + if "SUMMARY:" in line_str or "ERROR:" in line_str: + logger.info(f"Detected crash: {line_str}") + stats["crashes_found"] += 1 + + await proc.wait() + stats["execution_time"] = time.time() - start_time + + # Send final stats update + if stats_callback: + await stats_callback({ + "total_execs": stats["total_executions"], + "execs_per_sec": stats["total_executions"] / stats["execution_time"] if stats["execution_time"] > 0 else 0, + "crashes": stats["crashes_found"], + "coverage": stats["coverage"], + "corpus_size": stats["corpus_size"], + "elapsed_time": int(stats["execution_time"]) + }) + + logger.info( + f"Fuzzing completed: {stats['total_executions']} execs, " + f"{stats['crashes_found']} crashes" + ) + + except Exception as e: + logger.error(f"Fuzzing error: {e}") + + return findings, stats + + async def _parse_crash_artifacts( + self, + workspace: Path, + target_name: str + ) -> List[ModuleFinding]: + """ + Parse crash artifacts from fuzz/artifacts directory. + + Cargo-fuzz stores crashes in: fuzz/artifacts// + """ + findings = [] + artifacts_dir = workspace / "fuzz" / "artifacts" / target_name + + if not artifacts_dir.exists(): + logger.info("No crash artifacts found") + return findings + + # Find all crash files + for crash_file in artifacts_dir.glob("crash-*"): + try: + finding = await self._analyze_crash(workspace, target_name, crash_file) + if finding: + findings.append(finding) + except Exception as e: + logger.warning(f"Failed to analyze crash {crash_file}: {e}") + + logger.info(f"Parsed {len(findings)} crash artifacts") + return findings + + async def _analyze_crash( + self, + workspace: Path, + target_name: str, + crash_file: Path + ) -> Optional[ModuleFinding]: + """ + Analyze a single crash file. + + Runs cargo-fuzz with the crash input to reproduce and get stack trace. + """ + try: + # Read crash input + crash_input = crash_file.read_bytes() + + # Reproduce crash to get stack trace + cmd = [ + "cargo", "fuzz", "run", + target_name, + str(crash_file) + ] + + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=workspace, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + env={**os.environ, "RUST_BACKTRACE": "1"} + ) + + stdout, _ = await proc.communicate() + output = stdout.decode('utf-8', errors='ignore') + + # Parse stack trace and error type + error_type = "Unknown Crash" + stack_trace = output + + # Extract error type + if "SEGV" in output: + error_type = "Segmentation Fault" + severity = "critical" + elif "heap-use-after-free" in output: + error_type = "Use After Free" + severity = "critical" + elif "heap-buffer-overflow" in output: + error_type = "Heap Buffer Overflow" + severity = "critical" + elif "stack-buffer-overflow" in output: + error_type = "Stack Buffer Overflow" + severity = "high" + elif "panic" in output.lower(): + error_type = "Panic" + severity = "medium" + else: + severity = "high" + + # Create finding + finding = self.create_finding( + title=f"Crash: {error_type} in {target_name}", + description=f"Cargo-fuzz discovered a crash in target '{target_name}'. " + f"Error type: {error_type}. " + f"Input size: {len(crash_input)} bytes.", + severity=severity, + category="crash", + file_path=f"fuzz/fuzz_targets/{target_name}.rs", + code_snippet=stack_trace[:500], + recommendation="Review the crash details and fix the underlying bug. " + "Use AddressSanitizer to identify memory safety issues. " + "Consider adding bounds checks or using safer APIs.", + metadata={ + "error_type": error_type, + "crash_file": crash_file.name, + "input_size": len(crash_input), + "reproducer": crash_file.name, + "stack_trace": stack_trace + } + ) + + return finding + + except Exception as e: + logger.warning(f"Failed to analyze crash {crash_file}: {e}") + return None diff --git a/backend/toolbox/modules/reporter/sarif_reporter.py b/backend/toolbox/modules/reporter/sarif_reporter.py index e504462..2a8bec7 100644 --- a/backend/toolbox/modules/reporter/sarif_reporter.py +++ b/backend/toolbox/modules/reporter/sarif_reporter.py @@ -17,7 +17,6 @@ import logging from pathlib import Path from typing import Dict, Any, List from datetime import datetime -import json try: from toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding diff --git a/backend/toolbox/modules/scanner/file_scanner.py b/backend/toolbox/modules/scanner/file_scanner.py index 908ab7e..22de200 100644 --- a/backend/toolbox/modules/scanner/file_scanner.py +++ b/backend/toolbox/modules/scanner/file_scanner.py @@ -16,16 +16,16 @@ File Scanner Module - Scans and enumerates files in the workspace import logging import mimetypes from pathlib import Path -from typing import Dict, Any, List +from typing import Dict, Any import hashlib try: - from toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding + from toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult except ImportError: try: - from modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding + from modules.base import BaseModule, ModuleMetadata, ModuleResult except ImportError: - from src.toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult, ModuleFinding + from src.toolbox.modules.base import BaseModule, ModuleMetadata, ModuleResult logger = logging.getLogger(__name__) diff --git a/backend/toolbox/modules/secret_detection/__init__.py b/backend/toolbox/modules/secret_detection/__init__.py index fa66d4e..e3fc98e 100644 --- a/backend/toolbox/modules/secret_detection/__init__.py +++ b/backend/toolbox/modules/secret_detection/__init__.py @@ -7,6 +7,8 @@ in codebases and repositories. Available modules: - TruffleHog: Comprehensive secret detection with verification - Gitleaks: Git-specific secret scanning and leak detection +- GitGuardian: Enterprise secret detection using GitGuardian API +- LLM Secret Detector: AI-powered semantic secret detection """ # Copyright (c) 2025 FuzzingLabs # diff --git a/backend/toolbox/modules/secret_detection/gitleaks.py b/backend/toolbox/modules/secret_detection/gitleaks.py index 5bf2716..7005236 100644 --- a/backend/toolbox/modules/secret_detection/gitleaks.py +++ b/backend/toolbox/modules/secret_detection/gitleaks.py @@ -248,7 +248,8 @@ class GitleaksModule(BaseModule): rule_id = result.get("RuleID", "unknown") description = result.get("Description", "") file_path = result.get("File", "") - line_number = result.get("LineNumber", 0) + line_number = result.get("StartLine", 0) # Gitleaks outputs "StartLine", not "LineNumber" + line_end = result.get("EndLine", 0) secret = result.get("Secret", "") match_text = result.get("Match", "") @@ -278,6 +279,7 @@ class GitleaksModule(BaseModule): category="secret_leak", file_path=file_path if file_path else None, line_start=line_number if line_number > 0 else None, + line_end=line_end if line_end > 0 else None, code_snippet=match_text if match_text else secret, recommendation=self._get_leak_recommendation(rule_id), metadata={ diff --git a/backend/toolbox/modules/secret_detection/llm_secret_detector.py b/backend/toolbox/modules/secret_detection/llm_secret_detector.py new file mode 100644 index 0000000..3ba96f8 --- /dev/null +++ b/backend/toolbox/modules/secret_detection/llm_secret_detector.py @@ -0,0 +1,397 @@ +""" +LLM Secret Detection Module + +This module uses an LLM to detect secrets and sensitive information via semantic understanding. +""" +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + + +import logging +from pathlib import Path +from typing import Dict, Any, List + +from ..base import BaseModule, ModuleMetadata, ModuleFinding, ModuleResult +from . import register_module + +logger = logging.getLogger(__name__) + + +@register_module +class LLMSecretDetectorModule(BaseModule): + """ + LLM-based secret detection module using AI semantic analysis. + + Uses an LLM agent to identify secrets through natural language understanding, + potentially catching secrets that pattern-based tools miss. + """ + + def get_metadata(self) -> ModuleMetadata: + """Get module metadata""" + return ModuleMetadata( + name="llm_secret_detector", + version="1.0.0", + description="AI-powered secret detection using LLM semantic analysis", + author="FuzzForge Team", + category="secret_detection", + tags=["secrets", "llm", "ai", "semantic"], + input_schema={ + "type": "object", + "properties": { + "agent_url": { + "type": "string", + "default": "http://fuzzforge-task-agent:8000/a2a/litellm_agent", + "description": "A2A agent endpoint URL" + }, + "llm_model": { + "type": "string", + "default": "gpt-4o-mini", + "description": "LLM model to use" + }, + "llm_provider": { + "type": "string", + "default": "openai", + "description": "LLM provider (openai, anthropic, etc.)" + }, + "file_patterns": { + "type": "array", + "items": {"type": "string"}, + "default": ["*.py", "*.js", "*.ts", "*.java", "*.go", "*.env", "*.yaml", "*.yml", "*.json", "*.xml", "*.ini", "*.sql", "*.properties", "*.sh", "*.bat", "*.config", "*.conf", "*.toml", "*id_rsa*"], + "description": "File patterns to analyze" + }, + "max_files": { + "type": "integer", + "default": 20, + "description": "Maximum number of files to analyze" + }, + "max_file_size": { + "type": "integer", + "default": 30000, + "description": "Maximum file size in bytes (30KB default)" + }, + "timeout": { + "type": "integer", + "default": 45, + "description": "Timeout per file in seconds" + } + }, + "required": [] + }, + output_schema={ + "type": "object", + "properties": { + "findings": { + "type": "array", + "description": "Secrets identified by LLM" + } + } + } + ) + + def validate_config(self, config: Dict[str, Any]) -> bool: + """Validate module configuration""" + # Lazy import to avoid Temporal sandbox restrictions + try: + from fuzzforge_ai.a2a_wrapper import send_agent_task # noqa: F401 + except ImportError: + raise RuntimeError( + "A2A wrapper not available. Ensure fuzzforge_ai module is accessible." + ) + + agent_url = config.get("agent_url") + if not agent_url or not isinstance(agent_url, str): + raise ValueError("agent_url must be a valid URL string") + + max_files = config.get("max_files", 20) + if not isinstance(max_files, int) or max_files <= 0: + raise ValueError("max_files must be a positive integer") + + return True + + async def execute(self, config: Dict[str, Any], workspace: Path) -> ModuleResult: + """ + Execute LLM-based secret detection. + + Args: + config: Module configuration + workspace: Path to the workspace containing code to analyze + + Returns: + ModuleResult with secrets detected by LLM + """ + self.start_timer() + + logger.info(f"Starting LLM secret detection in workspace: {workspace}") + + # Extract configuration + agent_url = config.get("agent_url", "http://fuzzforge-task-agent:8000/a2a/litellm_agent") + llm_model = config.get("llm_model", "gpt-4o-mini") + llm_provider = config.get("llm_provider", "openai") + file_patterns = config.get("file_patterns", ["*.py", "*.js", "*.ts", "*.java", "*.go", "*.env", "*.yaml", "*.yml", "*.json", "*.xml", "*.ini", "*.sql", "*.properties", "*.sh", "*.bat", "*.config", "*.conf", "*.toml", "*id_rsa*", "*.txt"]) + max_files = config.get("max_files", 20) + max_file_size = config.get("max_file_size", 30000) + timeout = config.get("timeout", 30) # Reduced from 45s + + # Find files to analyze + # Skip files that are unlikely to contain secrets + skip_patterns = ['*.sarif', '*.md', '*.html', '*.css', '*.db', '*.sqlite'] + + files_to_analyze = [] + for pattern in file_patterns: + for file_path in workspace.rglob(pattern): + if file_path.is_file(): + try: + # Skip unlikely files + if any(file_path.match(skip) for skip in skip_patterns): + logger.debug(f"Skipping {file_path.name} (unlikely to have secrets)") + continue + + # Check file size + if file_path.stat().st_size > max_file_size: + logger.debug(f"Skipping {file_path} (too large)") + continue + + files_to_analyze.append(file_path) + + if len(files_to_analyze) >= max_files: + break + except Exception as e: + logger.warning(f"Error checking file {file_path}: {e}") + continue + + if len(files_to_analyze) >= max_files: + break + + logger.info(f"Found {len(files_to_analyze)} files to analyze for secrets") + + # Analyze each file with LLM + all_findings = [] + for file_path in files_to_analyze: + logger.info(f"Analyzing: {file_path.relative_to(workspace)}") + + try: + findings = await self._analyze_file_for_secrets( + file_path=file_path, + workspace=workspace, + agent_url=agent_url, + llm_model=llm_model, + llm_provider=llm_provider, + timeout=timeout + ) + all_findings.extend(findings) + + except Exception as e: + logger.error(f"Error analyzing {file_path}: {e}") + # Continue with next file + continue + + logger.info(f"LLM secret detection complete. Found {len(all_findings)} potential secrets.") + + # Create result + return self.create_result( + findings=all_findings, + status="success", + summary={ + "files_analyzed": len(files_to_analyze), + "total_secrets": len(all_findings), + "agent_url": agent_url, + "model": f"{llm_provider}/{llm_model}" + } + ) + + async def _analyze_file_for_secrets( + self, + file_path: Path, + workspace: Path, + agent_url: str, + llm_model: str, + llm_provider: str, + timeout: int + ) -> List[ModuleFinding]: + """Analyze a single file for secrets using LLM""" + + # Read file content + try: + with open(file_path, 'r', encoding='utf-8') as f: + code_content = f.read() + except Exception as e: + logger.error(f"Failed to read {file_path}: {e}") + return [] + + # Build specialized prompt for secret detection + system_prompt = ( + "You are a security expert specialized in detecting secrets and credentials in code. " + "Your job is to find REAL secrets that could be exploited. Be thorough and aggressive.\n\n" + "For each secret found, respond in this exact format:\n" + "SECRET_FOUND: [type like 'AWS Key', 'GitHub Token', 'Database Password']\n" + "SEVERITY: [critical/high/medium/low]\n" + "LINE: [exact line number]\n" + "CONFIDENCE: [high/medium/low]\n" + "DESCRIPTION: [brief explanation]\n\n" + "EXAMPLES of secrets to find:\n" + "1. API Keys: 'AKIA...', 'ghp_...', 'sk_live_...', 'SG.'\n" + "2. Tokens: Bearer tokens, OAuth tokens, JWT secrets\n" + "3. Passwords: Database passwords, admin passwords in configs\n" + "4. Connection Strings: mongodb://, postgres://, redis:// with credentials\n" + "5. Private Keys: -----BEGIN PRIVATE KEY-----, -----BEGIN RSA PRIVATE KEY-----\n" + "6. Cloud Credentials: AWS keys, GCP keys, Azure keys\n" + "7. Encryption Keys: AES keys, secret keys in config\n" + "8. Webhook URLs: URLs with tokens like hooks.slack.com/services/...\n\n" + "FIND EVERYTHING that looks like a real credential, password, key, or token.\n" + "DO NOT be overly cautious. Report anything suspicious.\n\n" + "If absolutely no secrets exist, respond with 'NO_SECRETS_FOUND'." + ) + + user_message = ( + f"Analyze this code for secrets and credentials:\n\n" + f"File: {file_path.relative_to(workspace)}\n\n" + f"```\n{code_content}\n```" + ) + + # Call LLM via A2A wrapper + try: + from fuzzforge_ai.a2a_wrapper import send_agent_task + + result = await send_agent_task( + url=agent_url, + model=llm_model, + provider=llm_provider, + prompt=system_prompt, + message=user_message, + context=f"secret_detection_{file_path.stem}", + timeout=float(timeout) + ) + + llm_response = result.text + + # Debug: Log LLM response + logger.debug(f"LLM response for {file_path.name}: {llm_response[:200]}...") + + except Exception as e: + logger.error(f"A2A call failed for {file_path}: {e}") + return [] + + # Parse LLM response into findings + findings = self._parse_llm_response( + llm_response=llm_response, + file_path=file_path, + workspace=workspace + ) + + if findings: + logger.info(f"Found {len(findings)} secrets in {file_path.name}") + else: + logger.debug(f"No secrets found in {file_path.name}. Response: {llm_response[:500]}") + + return findings + + def _parse_llm_response( + self, + llm_response: str, + file_path: Path, + workspace: Path + ) -> List[ModuleFinding]: + """Parse LLM response into structured findings""" + + if "NO_SECRETS_FOUND" in llm_response: + return [] + + findings = [] + relative_path = str(file_path.relative_to(workspace)) + + # Simple parser for the expected format + lines = llm_response.split('\n') + current_secret = {} + + for line in lines: + line = line.strip() + + if line.startswith("SECRET_FOUND:"): + # Save previous secret if exists + if current_secret: + findings.append(self._create_secret_finding(current_secret, relative_path)) + current_secret = {"type": line.replace("SECRET_FOUND:", "").strip()} + + elif line.startswith("SEVERITY:"): + severity = line.replace("SEVERITY:", "").strip().lower() + current_secret["severity"] = severity + + elif line.startswith("LINE:"): + line_num = line.replace("LINE:", "").strip() + try: + current_secret["line"] = int(line_num) + except ValueError: + current_secret["line"] = None + + elif line.startswith("CONFIDENCE:"): + confidence = line.replace("CONFIDENCE:", "").strip().lower() + current_secret["confidence"] = confidence + + elif line.startswith("DESCRIPTION:"): + current_secret["description"] = line.replace("DESCRIPTION:", "").strip() + + # Save last secret + if current_secret: + findings.append(self._create_secret_finding(current_secret, relative_path)) + + return findings + + def _create_secret_finding(self, secret: Dict[str, Any], file_path: str) -> ModuleFinding: + """Create a ModuleFinding from parsed secret""" + + severity_map = { + "critical": "critical", + "high": "high", + "medium": "medium", + "low": "low" + } + + severity = severity_map.get(secret.get("severity", "medium"), "medium") + confidence = secret.get("confidence", "medium") + + # Adjust severity based on confidence + if confidence == "low" and severity == "critical": + severity = "high" + elif confidence == "low" and severity == "high": + severity = "medium" + + # Create finding + title = f"LLM detected secret: {secret.get('type', 'Unknown secret')}" + description = secret.get("description", "An LLM identified this as a potential secret.") + description += f"\n\nConfidence: {confidence}" + + return self.create_finding( + title=title, + description=description, + severity=severity, + category="secret_detection", + file_path=file_path, + line_start=secret.get("line"), + recommendation=self._get_secret_recommendation(secret.get("type", "")), + metadata={ + "tool": "llm-secret-detector", + "secret_type": secret.get("type", "unknown"), + "confidence": confidence, + "detection_method": "semantic-analysis" + } + ) + + def _get_secret_recommendation(self, secret_type: str) -> str: + """Get remediation recommendation for detected secret""" + return ( + f"A potential {secret_type} was detected by AI analysis. " + f"Verify whether this is a real secret or a false positive. " + f"If real: (1) Revoke the credential immediately, " + f"(2) Remove from codebase and Git history, " + f"(3) Rotate to a new secret, " + f"(4) Use secret management tools for storage. " + f"Implement pre-commit hooks to prevent future leaks." + ) diff --git a/backend/toolbox/modules/secret_detection/trufflehog.py b/backend/toolbox/modules/secret_detection/trufflehog.py index 733482e..6c68e99 100644 --- a/backend/toolbox/modules/secret_detection/trufflehog.py +++ b/backend/toolbox/modules/secret_detection/trufflehog.py @@ -61,11 +61,6 @@ class TruffleHogModule(BaseModule): "items": {"type": "string"}, "description": "Specific detectors to exclude" }, - "max_depth": { - "type": "integer", - "default": 10, - "description": "Maximum directory depth to scan" - }, "concurrency": { "type": "integer", "default": 10, @@ -100,11 +95,6 @@ class TruffleHogModule(BaseModule): if not isinstance(concurrency, int) or concurrency < 1 or concurrency > 50: raise ValueError("Concurrency must be between 1 and 50") - # Check max_depth bounds - max_depth = config.get("max_depth", 10) - if not isinstance(max_depth, int) or max_depth < 1 or max_depth > 20: - raise ValueError("Max depth must be between 1 and 20") - return True async def execute(self, config: Dict[str, Any], workspace: Path) -> ModuleResult: @@ -124,6 +114,9 @@ class TruffleHogModule(BaseModule): # Add verification flag if config.get("verify", False): cmd.append("--verify") + else: + # Explicitly disable verification to get all unverified secrets + cmd.append("--no-verification") # Add JSON output cmd.extend(["--json", "--no-update"]) @@ -131,9 +124,6 @@ class TruffleHogModule(BaseModule): # Add concurrency cmd.extend(["--concurrency", str(config.get("concurrency", 10))]) - # Add max depth - cmd.extend(["--max-depth", str(config.get("max_depth", 10))]) - # Add include/exclude detectors if config.get("include_detectors"): cmd.extend(["--include-detectors", ",".join(config["include_detectors"])]) diff --git a/backend/toolbox/workflows/atheris_fuzzing/__init__.py b/backend/toolbox/workflows/atheris_fuzzing/__init__.py new file mode 100644 index 0000000..38b1648 --- /dev/null +++ b/backend/toolbox/workflows/atheris_fuzzing/__init__.py @@ -0,0 +1,9 @@ +""" +Atheris Fuzzing Workflow + +Fuzzes user-provided Python code using Atheris. +""" + +from .workflow import AtherisFuzzingWorkflow + +__all__ = ["AtherisFuzzingWorkflow"] diff --git a/backend/toolbox/workflows/atheris_fuzzing/activities.py b/backend/toolbox/workflows/atheris_fuzzing/activities.py new file mode 100644 index 0000000..2ed31b7 --- /dev/null +++ b/backend/toolbox/workflows/atheris_fuzzing/activities.py @@ -0,0 +1,122 @@ +""" +Atheris Fuzzing Workflow Activities + +Activities specific to the Atheris fuzzing workflow. +""" + +import logging +import sys +from datetime import datetime +from pathlib import Path +from typing import Dict, Any +import os + +import httpx +from temporalio import activity + +# Configure logging +logger = logging.getLogger(__name__) + +# Add toolbox to path for module imports +sys.path.insert(0, '/app/toolbox') + + +@activity.defn(name="fuzz_with_atheris") +async def fuzz_activity(workspace_path: str, config: dict) -> dict: + """ + Fuzzing activity using the AtherisFuzzer module on user code. + + This activity: + 1. Imports the reusable AtherisFuzzer module + 2. Sets up real-time stats callback + 3. Executes fuzzing on user's TestOneInput() function + 4. Returns findings as ModuleResult + + Args: + workspace_path: Path to the workspace directory (user's uploaded code) + config: Fuzzer configuration (target_file, max_iterations, timeout_seconds) + + Returns: + Fuzzer results dictionary (findings, summary, metadata) + """ + logger.info(f"Activity: fuzz_with_atheris (workspace={workspace_path})") + + try: + # Import reusable AtherisFuzzer module + from modules.fuzzer import AtherisFuzzer + + workspace = Path(workspace_path) + if not workspace.exists(): + raise FileNotFoundError(f"Workspace not found: {workspace_path}") + + # Get activity info for real-time stats + info = activity.info() + run_id = info.workflow_id + + # Define stats callback for real-time monitoring + async def stats_callback(stats_data: Dict[str, Any]): + """Callback for live fuzzing statistics""" + try: + # Prepare stats payload for backend + coverage_value = stats_data.get("coverage", 0) + logger.info(f"COVERAGE_DEBUG: coverage from stats_data = {coverage_value}") + + stats_payload = { + "run_id": run_id, + "workflow": "atheris_fuzzing", + "executions": stats_data.get("total_execs", 0), + "executions_per_sec": stats_data.get("execs_per_sec", 0.0), + "crashes": stats_data.get("crashes", 0), + "unique_crashes": stats_data.get("crashes", 0), + "coverage": coverage_value, + "corpus_size": stats_data.get("corpus_size", 0), + "elapsed_time": stats_data.get("elapsed_time", 0), + "last_crash_time": None + } + + # POST stats to backend API for real-time monitoring + backend_url = os.getenv("BACKEND_URL", "http://backend:8000") + async with httpx.AsyncClient(timeout=5.0) as client: + try: + await client.post( + f"{backend_url}/fuzzing/{run_id}/stats", + json=stats_payload + ) + except Exception as http_err: + logger.debug(f"Failed to post stats to backend: {http_err}") + + # Also log for debugging + logger.info("LIVE_STATS", extra={ + "stats_type": "fuzzing_live_update", + "workflow_type": "atheris_fuzzing", + "run_id": run_id, + "executions": stats_data.get("total_execs", 0), + "executions_per_sec": stats_data.get("execs_per_sec", 0.0), + "crashes": stats_data.get("crashes", 0), + "corpus_size": stats_data.get("corpus_size", 0), + "coverage": stats_data.get("coverage", 0.0), + "elapsed_time": stats_data.get("elapsed_time", 0), + "timestamp": datetime.utcnow().isoformat() + }) + except Exception as e: + logger.warning(f"Error in stats callback: {e}") + + # Add stats callback and run_id to config + config["stats_callback"] = stats_callback + config["run_id"] = run_id + + # Execute the fuzzer module + fuzzer = AtherisFuzzer() + result = await fuzzer.execute(config, workspace) + + logger.info( + f"āœ“ Fuzzing completed: " + f"{result.summary.get('total_executions', 0)} executions, " + f"{result.summary.get('crashes_found', 0)} crashes" + ) + + return result.dict() + + except Exception as e: + logger.error(f"Fuzzing failed: {e}", exc_info=True) + raise diff --git a/backend/toolbox/workflows/atheris_fuzzing/metadata.yaml b/backend/toolbox/workflows/atheris_fuzzing/metadata.yaml new file mode 100644 index 0000000..b079804 --- /dev/null +++ b/backend/toolbox/workflows/atheris_fuzzing/metadata.yaml @@ -0,0 +1,65 @@ +name: atheris_fuzzing +version: "1.0.0" +vertical: python +description: "Fuzz Python code using Atheris with real-time monitoring. Automatically discovers and fuzzes TestOneInput() functions in user code." +author: "FuzzForge Team" +tags: + - "fuzzing" + - "atheris" + - "python" + - "coverage" + - "security" + +# Workspace isolation mode (system-level configuration) +# - "isolated" (default): Each workflow run gets its own isolated workspace (safe for concurrent fuzzing) +# - "shared": All runs share the same workspace (for read-only analysis workflows) +# - "copy-on-write": Download once, copy for each run (balances performance and isolation) +workspace_isolation: "isolated" + +default_parameters: + target_file: null + max_iterations: 1000000 + timeout_seconds: 1800 + +parameters: + type: object + properties: + target_file: + type: string + description: "Python file with TestOneInput() function (auto-discovered if not specified)" + max_iterations: + type: integer + default: 1000000 + description: "Maximum fuzzing iterations" + timeout_seconds: + type: integer + default: 1800 + description: "Fuzzing timeout in seconds (30 minutes)" + +output_schema: + type: object + properties: + findings: + type: array + description: "Crashes and vulnerabilities found during fuzzing" + items: + type: object + properties: + title: + type: string + severity: + type: string + category: + type: string + metadata: + type: object + summary: + type: object + description: "Fuzzing execution summary" + properties: + total_executions: + type: integer + crashes_found: + type: integer + execution_time: + type: number diff --git a/backend/toolbox/workflows/atheris_fuzzing/workflow.py b/backend/toolbox/workflows/atheris_fuzzing/workflow.py new file mode 100644 index 0000000..a9b0cad --- /dev/null +++ b/backend/toolbox/workflows/atheris_fuzzing/workflow.py @@ -0,0 +1,175 @@ +""" +Atheris Fuzzing Workflow - Temporal Version + +Fuzzes user-provided Python code using Atheris with real-time monitoring. +""" + +from datetime import timedelta +from typing import Dict, Any, Optional + +from temporalio import workflow +from temporalio.common import RetryPolicy + +# Import for type hints (will be executed by worker) +with workflow.unsafe.imports_passed_through(): + import logging + +logger = logging.getLogger(__name__) + + +@workflow.defn +class AtherisFuzzingWorkflow: + """ + Fuzz Python code using Atheris. + + User workflow: + 1. User runs: ff workflow run atheris_fuzzing . + 2. CLI uploads project to MinIO + 3. Worker downloads project + 4. Worker fuzzes TestOneInput() function + 5. Crashes reported as findings + """ + + @workflow.run + async def run( + self, + target_id: str, # MinIO UUID of uploaded user code + target_file: Optional[str] = None, # Optional: specific file to fuzz + max_iterations: int = 1000000, + timeout_seconds: int = 1800 # 30 minutes default for fuzzing + ) -> Dict[str, Any]: + """ + Main workflow execution. + + Args: + target_id: UUID of the uploaded target in MinIO + target_file: Optional specific Python file with TestOneInput() (auto-discovered if None) + max_iterations: Maximum fuzzing iterations + timeout_seconds: Fuzzing timeout in seconds + + Returns: + Dictionary containing findings and summary + """ + workflow_id = workflow.info().workflow_id + + workflow.logger.info( + f"Starting AtherisFuzzingWorkflow " + f"(workflow_id={workflow_id}, target_id={target_id}, " + f"target_file={target_file or 'auto-discover'}, max_iterations={max_iterations}, " + f"timeout_seconds={timeout_seconds})" + ) + + results = { + "workflow_id": workflow_id, + "target_id": target_id, + "status": "running", + "steps": [] + } + + try: + # Get run ID for workspace isolation + run_id = workflow.info().run_id + + # Step 1: Download user's project from MinIO + workflow.logger.info("Step 1: Downloading user code from MinIO") + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "isolated"], # target_id, run_id, workspace_isolation + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["steps"].append({ + "step": "download_target", + "status": "success", + "target_path": target_path + }) + workflow.logger.info(f"āœ“ User code downloaded to: {target_path}") + + # Step 2: Run Atheris fuzzing + workflow.logger.info("Step 2: Running Atheris fuzzing") + + # Use defaults if parameters are None + actual_max_iterations = max_iterations if max_iterations is not None else 1000000 + actual_timeout_seconds = timeout_seconds if timeout_seconds is not None else 1800 + + fuzz_config = { + "target_file": target_file, + "max_iterations": actual_max_iterations, + "timeout_seconds": actual_timeout_seconds + } + + fuzz_results = await workflow.execute_activity( + "fuzz_with_atheris", + args=[target_path, fuzz_config], + start_to_close_timeout=timedelta(seconds=actual_timeout_seconds + 60), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=1 # Fuzzing shouldn't retry + ) + ) + + results["steps"].append({ + "step": "fuzzing", + "status": "success", + "executions": fuzz_results.get("summary", {}).get("total_executions", 0), + "crashes": fuzz_results.get("summary", {}).get("crashes_found", 0) + }) + workflow.logger.info( + f"āœ“ Fuzzing completed: " + f"{fuzz_results.get('summary', {}).get('total_executions', 0)} executions, " + f"{fuzz_results.get('summary', {}).get('crashes_found', 0)} crashes" + ) + + # Step 3: Upload results to MinIO + workflow.logger.info("Step 3: Uploading results") + try: + results_url = await workflow.execute_activity( + "upload_results", + args=[workflow_id, fuzz_results, "json"], + start_to_close_timeout=timedelta(minutes=2) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + except Exception as e: + workflow.logger.warning(f"Failed to upload results: {e}") + results["results_url"] = None + + # Step 4: Cleanup cache + workflow.logger.info("Step 4: Cleaning up cache") + try: + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "isolated"], # target_path, workspace_isolation + start_to_close_timeout=timedelta(minutes=1) + ) + workflow.logger.info("āœ“ Cache cleaned up") + except Exception as e: + workflow.logger.warning(f"Cache cleanup failed: {e}") + + # Mark workflow as successful + results["status"] = "success" + results["findings"] = fuzz_results.get("findings", []) + results["summary"] = fuzz_results.get("summary", {}) + results["sarif"] = fuzz_results.get("sarif") or {} + workflow.logger.info( + f"āœ“ Workflow completed successfully: {workflow_id} " + f"({results['summary'].get('crashes_found', 0)} crashes found)" + ) + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + results["steps"].append({ + "step": "error", + "status": "failed", + "error": str(e) + }) + raise diff --git a/backend/toolbox/workflows/cargo_fuzzing/__init__.py b/backend/toolbox/workflows/cargo_fuzzing/__init__.py new file mode 100644 index 0000000..d496e88 --- /dev/null +++ b/backend/toolbox/workflows/cargo_fuzzing/__init__.py @@ -0,0 +1,5 @@ +"""Cargo Fuzzing Workflow""" + +from .workflow import CargoFuzzingWorkflow + +__all__ = ["CargoFuzzingWorkflow"] diff --git a/backend/toolbox/workflows/cargo_fuzzing/activities.py b/backend/toolbox/workflows/cargo_fuzzing/activities.py new file mode 100644 index 0000000..e23e929 --- /dev/null +++ b/backend/toolbox/workflows/cargo_fuzzing/activities.py @@ -0,0 +1,203 @@ +""" +Cargo Fuzzing Workflow Activities + +Activities specific to the cargo-fuzz fuzzing workflow. +""" + +import logging +import sys +from datetime import datetime +from pathlib import Path +from typing import Dict, Any +import os + +import httpx +from temporalio import activity + +# Configure logging +logger = logging.getLogger(__name__) + +# Add toolbox to path for module imports +sys.path.insert(0, '/app/toolbox') + + +@activity.defn(name="fuzz_with_cargo") +async def fuzz_activity(workspace_path: str, config: dict) -> dict: + """ + Fuzzing activity using the CargoFuzzer module on user code. + + This activity: + 1. Imports the reusable CargoFuzzer module + 2. Sets up real-time stats callback + 3. Executes fuzzing on user's fuzz_target!() functions + 4. Returns findings as ModuleResult + + Args: + workspace_path: Path to the workspace directory (user's uploaded Rust project) + config: Fuzzer configuration (target_name, max_iterations, timeout_seconds, sanitizer) + + Returns: + Fuzzer results dictionary (findings, summary, metadata) + """ + logger.info(f"Activity: fuzz_with_cargo (workspace={workspace_path})") + + try: + # Import reusable CargoFuzzer module + from modules.fuzzer import CargoFuzzer + + workspace = Path(workspace_path) + if not workspace.exists(): + raise FileNotFoundError(f"Workspace not found: {workspace_path}") + + # Get activity info for real-time stats + info = activity.info() + run_id = info.workflow_id + + # Define stats callback for real-time monitoring + async def stats_callback(stats_data: Dict[str, Any]): + """Callback for live fuzzing statistics""" + try: + # Prepare stats payload for backend + coverage_value = stats_data.get("coverage", 0) + + stats_payload = { + "run_id": run_id, + "workflow": "cargo_fuzzing", + "executions": stats_data.get("total_execs", 0), + "executions_per_sec": stats_data.get("execs_per_sec", 0.0), + "crashes": stats_data.get("crashes", 0), + "unique_crashes": stats_data.get("crashes", 0), + "coverage": coverage_value, + "corpus_size": stats_data.get("corpus_size", 0), + "elapsed_time": stats_data.get("elapsed_time", 0), + "last_crash_time": None + } + + # POST stats to backend API for real-time monitoring + backend_url = os.getenv("BACKEND_URL", "http://backend:8000") + async with httpx.AsyncClient(timeout=5.0) as client: + try: + await client.post( + f"{backend_url}/fuzzing/{run_id}/stats", + json=stats_payload + ) + except Exception as http_err: + logger.debug(f"Failed to post stats to backend: {http_err}") + + # Also log for debugging + logger.info("LIVE_STATS", extra={ + "stats_type": "fuzzing_live_update", + "workflow_type": "cargo_fuzzing", + "run_id": run_id, + "executions": stats_data.get("total_execs", 0), + "executions_per_sec": stats_data.get("execs_per_sec", 0.0), + "crashes": stats_data.get("crashes", 0), + "corpus_size": stats_data.get("corpus_size", 0), + "coverage": stats_data.get("coverage", 0.0), + "elapsed_time": stats_data.get("elapsed_time", 0), + "timestamp": datetime.utcnow().isoformat() + }) + + except Exception as e: + logger.error(f"Stats callback error: {e}") + + # Initialize CargoFuzzer module + fuzzer = CargoFuzzer() + + # Execute fuzzing with stats callback + module_result = await fuzzer.execute( + config=config, + workspace=workspace, + stats_callback=stats_callback + ) + + # Convert ModuleResult to dictionary + result_dict = { + "findings": [], + "summary": module_result.summary, + "metadata": module_result.metadata, + "status": module_result.status, + "error": module_result.error + } + + # Convert findings to dict format + for finding in module_result.findings: + finding_dict = { + "id": finding.id, + "title": finding.title, + "description": finding.description, + "severity": finding.severity, + "category": finding.category, + "file_path": finding.file_path, + "line_start": finding.line_start, + "line_end": finding.line_end, + "code_snippet": finding.code_snippet, + "recommendation": finding.recommendation, + "metadata": finding.metadata + } + result_dict["findings"].append(finding_dict) + + # Generate SARIF report from findings + if module_result.findings: + # Convert findings to SARIF format + severity_map = { + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note" + } + + results = [] + for finding in module_result.findings: + result = { + "ruleId": finding.metadata.get("rule_id", finding.category), + "level": severity_map.get(finding.severity, "warning"), + "message": {"text": finding.description}, + "locations": [] + } + + if finding.file_path: + location = { + "physicalLocation": { + "artifactLocation": {"uri": finding.file_path}, + "region": { + "startLine": finding.line_start or 1, + "endLine": finding.line_end or finding.line_start or 1 + } + } + } + result["locations"].append(location) + + results.append(result) + + result_dict["sarif"] = { + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [{ + "tool": { + "driver": { + "name": "cargo-fuzz", + "version": "0.11.2" + } + }, + "results": results + }] + } + else: + result_dict["sarif"] = { + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [] + } + + logger.info( + f"Fuzzing activity completed: {len(module_result.findings)} crashes found, " + f"{module_result.summary.get('total_executions', 0)} executions" + ) + + return result_dict + + except Exception as e: + logger.error(f"Fuzzing activity failed: {e}", exc_info=True) + raise diff --git a/backend/toolbox/workflows/cargo_fuzzing/metadata.yaml b/backend/toolbox/workflows/cargo_fuzzing/metadata.yaml new file mode 100644 index 0000000..39ff622 --- /dev/null +++ b/backend/toolbox/workflows/cargo_fuzzing/metadata.yaml @@ -0,0 +1,71 @@ +name: cargo_fuzzing +version: "1.0.0" +vertical: rust +description: "Fuzz Rust code using cargo-fuzz with real-time monitoring. Automatically discovers and fuzzes fuzz_target!() functions in user code." +author: "FuzzForge Team" +tags: + - "fuzzing" + - "cargo-fuzz" + - "rust" + - "libfuzzer" + - "memory-safety" + +# Workspace isolation mode (system-level configuration) +# - "isolated" (default): Each workflow run gets its own isolated workspace (safe for concurrent fuzzing) +# - "shared": All runs share the same workspace (for read-only analysis workflows) +# - "copy-on-write": Download once, copy for each run (balances performance and isolation) +workspace_isolation: "isolated" + +default_parameters: + target_name: null + max_iterations: 1000000 + timeout_seconds: 1800 + sanitizer: "address" + +parameters: + type: object + properties: + target_name: + type: string + description: "Fuzz target name from fuzz/fuzz_targets/ (auto-discovered if not specified)" + max_iterations: + type: integer + default: 1000000 + description: "Maximum fuzzing iterations" + timeout_seconds: + type: integer + default: 1800 + description: "Fuzzing timeout in seconds (30 minutes)" + sanitizer: + type: string + enum: ["address", "memory", "undefined"] + default: "address" + description: "Sanitizer to use (address, memory, undefined)" + +output_schema: + type: object + properties: + findings: + type: array + description: "Crashes and memory safety issues found during fuzzing" + items: + type: object + properties: + title: + type: string + severity: + type: string + category: + type: string + metadata: + type: object + summary: + type: object + description: "Fuzzing execution summary" + properties: + total_executions: + type: integer + crashes_found: + type: integer + execution_time: + type: number diff --git a/backend/toolbox/workflows/cargo_fuzzing/workflow.py b/backend/toolbox/workflows/cargo_fuzzing/workflow.py new file mode 100644 index 0000000..5581ee0 --- /dev/null +++ b/backend/toolbox/workflows/cargo_fuzzing/workflow.py @@ -0,0 +1,180 @@ +""" +Cargo Fuzzing Workflow - Temporal Version + +Fuzzes user-provided Rust code using cargo-fuzz with real-time monitoring. +""" + +from datetime import timedelta +from typing import Dict, Any, Optional + +from temporalio import workflow +from temporalio.common import RetryPolicy + +# Import for type hints (will be executed by worker) +with workflow.unsafe.imports_passed_through(): + import logging + +logger = logging.getLogger(__name__) + + +@workflow.defn +class CargoFuzzingWorkflow: + """ + Fuzz Rust code using cargo-fuzz (libFuzzer). + + User workflow: + 1. User runs: ff workflow run cargo_fuzzing . + 2. CLI uploads Rust project to MinIO + 3. Worker downloads project + 4. Worker discovers fuzz targets in fuzz/fuzz_targets/ + 5. Worker fuzzes the target with cargo-fuzz + 6. Crashes reported as findings + """ + + @workflow.run + async def run( + self, + target_id: str, # MinIO UUID of uploaded user code + target_name: Optional[str] = None, # Optional: specific fuzz target name + max_iterations: int = 1000000, + timeout_seconds: int = 1800, # 30 minutes default for fuzzing + sanitizer: str = "address" + ) -> Dict[str, Any]: + """ + Main workflow execution. + + Args: + target_id: UUID of the uploaded target in MinIO + target_name: Optional specific fuzz target name (auto-discovered if None) + max_iterations: Maximum fuzzing iterations + timeout_seconds: Fuzzing timeout in seconds + sanitizer: Sanitizer to use (address, memory, undefined) + + Returns: + Dictionary containing findings and summary + """ + workflow_id = workflow.info().workflow_id + + workflow.logger.info( + f"Starting CargoFuzzingWorkflow " + f"(workflow_id={workflow_id}, target_id={target_id}, " + f"target_name={target_name or 'auto-discover'}, max_iterations={max_iterations}, " + f"timeout_seconds={timeout_seconds}, sanitizer={sanitizer})" + ) + + results = { + "workflow_id": workflow_id, + "target_id": target_id, + "status": "running", + "steps": [] + } + + try: + # Get run ID for workspace isolation + run_id = workflow.info().run_id + + # Step 1: Download user's Rust project from MinIO + workflow.logger.info("Step 1: Downloading user code from MinIO") + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "isolated"], # target_id, run_id, workspace_isolation + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["steps"].append({ + "step": "download_target", + "status": "success", + "target_path": target_path + }) + workflow.logger.info(f"āœ“ User code downloaded to: {target_path}") + + # Step 2: Run cargo-fuzz + workflow.logger.info("Step 2: Running cargo-fuzz") + + # Use defaults if parameters are None + actual_max_iterations = max_iterations if max_iterations is not None else 1000000 + actual_timeout_seconds = timeout_seconds if timeout_seconds is not None else 1800 + actual_sanitizer = sanitizer if sanitizer is not None else "address" + + fuzz_config = { + "target_name": target_name, + "max_iterations": actual_max_iterations, + "timeout_seconds": actual_timeout_seconds, + "sanitizer": actual_sanitizer + } + + fuzz_results = await workflow.execute_activity( + "fuzz_with_cargo", + args=[target_path, fuzz_config], + start_to_close_timeout=timedelta(seconds=actual_timeout_seconds + 120), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=1 # Fuzzing shouldn't retry + ) + ) + + results["steps"].append({ + "step": "fuzzing", + "status": "success", + "executions": fuzz_results.get("summary", {}).get("total_executions", 0), + "crashes": fuzz_results.get("summary", {}).get("crashes_found", 0) + }) + workflow.logger.info( + f"āœ“ Fuzzing completed: " + f"{fuzz_results.get('summary', {}).get('total_executions', 0)} executions, " + f"{fuzz_results.get('summary', {}).get('crashes_found', 0)} crashes" + ) + + # Step 3: Upload results to MinIO + workflow.logger.info("Step 3: Uploading results") + try: + results_url = await workflow.execute_activity( + "upload_results", + args=[workflow_id, fuzz_results, "json"], + start_to_close_timeout=timedelta(minutes=2) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + except Exception as e: + workflow.logger.warning(f"Failed to upload results: {e}") + results["results_url"] = None + + # Step 4: Cleanup cache + workflow.logger.info("Step 4: Cleaning up cache") + try: + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "isolated"], # target_path, workspace_isolation + start_to_close_timeout=timedelta(minutes=1) + ) + workflow.logger.info("āœ“ Cache cleaned up") + except Exception as e: + workflow.logger.warning(f"Cache cleanup failed: {e}") + + # Mark workflow as successful + results["status"] = "success" + results["findings"] = fuzz_results.get("findings", []) + results["summary"] = fuzz_results.get("summary", {}) + results["sarif"] = fuzz_results.get("sarif") or {} + workflow.logger.info( + f"āœ“ Workflow completed successfully: {workflow_id} " + f"({results['summary'].get('crashes_found', 0)} crashes found)" + ) + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + results["steps"].append({ + "step": "error", + "status": "failed", + "error": str(e) + }) + raise diff --git a/backend/toolbox/workflows/comprehensive/secret_detection_scan/Dockerfile b/backend/toolbox/workflows/comprehensive/secret_detection_scan/Dockerfile deleted file mode 100644 index 96a6761..0000000 --- a/backend/toolbox/workflows/comprehensive/secret_detection_scan/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -# Secret Detection Workflow Dockerfile -FROM prefecthq/prefect:3-python3.11 - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - curl \ - wget \ - git \ - ca-certificates \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -# Install TruffleHog (use direct binary download to avoid install script issues) -RUN curl -sSfL "https://github.com/trufflesecurity/trufflehog/releases/download/v3.63.2/trufflehog_3.63.2_linux_amd64.tar.gz" -o trufflehog.tar.gz \ - && tar -xzf trufflehog.tar.gz \ - && mv trufflehog /usr/local/bin/ \ - && rm trufflehog.tar.gz - -# Install Gitleaks (use specific version to avoid API rate limiting) -RUN wget https://github.com/gitleaks/gitleaks/releases/download/v8.18.2/gitleaks_8.18.2_linux_x64.tar.gz \ - && tar -xzf gitleaks_8.18.2_linux_x64.tar.gz \ - && mv gitleaks /usr/local/bin/ \ - && rm gitleaks_8.18.2_linux_x64.tar.gz - -# Verify installations -RUN trufflehog --version && gitleaks version - -# Set working directory -WORKDIR /opt/prefect - -# Create toolbox directory structure -RUN mkdir -p /opt/prefect/toolbox - -# Set environment variables -ENV PYTHONPATH=/opt/prefect/toolbox:/opt/prefect/toolbox/workflows -ENV WORKFLOW_NAME=secret_detection_scan - -# The toolbox code will be mounted at runtime from the backend container -# This includes: -# - /opt/prefect/toolbox/modules/base.py -# - /opt/prefect/toolbox/modules/secret_detection/ (TruffleHog, Gitleaks modules) -# - /opt/prefect/toolbox/modules/reporter/ (SARIF reporter) -# - /opt/prefect/toolbox/workflows/comprehensive/secret_detection_scan/ -VOLUME /opt/prefect/toolbox - -# Set working directory for execution -WORKDIR /opt/prefect \ No newline at end of file diff --git a/backend/toolbox/workflows/comprehensive/secret_detection_scan/Dockerfile.self-contained b/backend/toolbox/workflows/comprehensive/secret_detection_scan/Dockerfile.self-contained deleted file mode 100644 index fae0243..0000000 --- a/backend/toolbox/workflows/comprehensive/secret_detection_scan/Dockerfile.self-contained +++ /dev/null @@ -1,58 +0,0 @@ -# Secret Detection Workflow Dockerfile - Self-Contained Version -# This version copies all required modules into the image for complete isolation -FROM prefecthq/prefect:3-python3.11 - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - curl \ - wget \ - git \ - ca-certificates \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -# Install TruffleHog -RUN curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin - -# Install Gitleaks -RUN wget https://github.com/gitleaks/gitleaks/releases/latest/download/gitleaks_linux_x64.tar.gz \ - && tar -xzf gitleaks_linux_x64.tar.gz \ - && mv gitleaks /usr/local/bin/ \ - && rm gitleaks_linux_x64.tar.gz - -# Verify installations -RUN trufflehog --version && gitleaks version - -# Set working directory -WORKDIR /opt/prefect - -# Create directory structure -RUN mkdir -p /opt/prefect/toolbox/modules/secret_detection \ - /opt/prefect/toolbox/modules/reporter \ - /opt/prefect/toolbox/workflows/comprehensive/secret_detection_scan - -# Copy the base module and required modules -COPY toolbox/modules/base.py /opt/prefect/toolbox/modules/base.py -COPY toolbox/modules/__init__.py /opt/prefect/toolbox/modules/__init__.py -COPY toolbox/modules/secret_detection/ /opt/prefect/toolbox/modules/secret_detection/ -COPY toolbox/modules/reporter/ /opt/prefect/toolbox/modules/reporter/ - -# Copy the workflow code -COPY toolbox/workflows/comprehensive/secret_detection_scan/ /opt/prefect/toolbox/workflows/comprehensive/secret_detection_scan/ - -# Copy toolbox init files -COPY toolbox/__init__.py /opt/prefect/toolbox/__init__.py -COPY toolbox/workflows/__init__.py /opt/prefect/toolbox/workflows/__init__.py -COPY toolbox/workflows/comprehensive/__init__.py /opt/prefect/toolbox/workflows/comprehensive/__init__.py - -# Install Python dependencies for the modules -RUN pip install --no-cache-dir \ - pydantic \ - asyncio - -# Set environment variables -ENV PYTHONPATH=/opt/prefect/toolbox:/opt/prefect/toolbox/workflows -ENV WORKFLOW_NAME=secret_detection_scan - -# Set default command (can be overridden) -CMD ["python", "-m", "toolbox.workflows.comprehensive.secret_detection_scan.workflow"] \ No newline at end of file diff --git a/backend/toolbox/workflows/comprehensive/secret_detection_scan/README.md b/backend/toolbox/workflows/comprehensive/secret_detection_scan/README.md deleted file mode 100644 index 51e99a2..0000000 --- a/backend/toolbox/workflows/comprehensive/secret_detection_scan/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# Secret Detection Scan Workflow - -This workflow performs comprehensive secret detection using multiple industry-standard tools: - -- **TruffleHog**: Comprehensive secret detection with verification capabilities -- **Gitleaks**: Git-specific secret scanning and leak detection - -## Features - -- **Parallel Execution**: Runs TruffleHog and Gitleaks concurrently for faster results -- **Deduplication**: Automatically removes duplicate findings across tools -- **SARIF Output**: Generates standardized SARIF reports for integration with security tools -- **Configurable**: Supports extensive configuration for both tools - -## Dependencies - -### Required Modules -- `toolbox.modules.secret_detection.trufflehog` -- `toolbox.modules.secret_detection.gitleaks` -- `toolbox.modules.reporter` (SARIF reporter) -- `toolbox.modules.base` (Base module interface) - -### External Tools -- TruffleHog v3.63.2+ -- Gitleaks v8.18.0+ - -## Docker Deployment - -This workflow provides two Docker deployment approaches: - -### 1. Volume-Based Approach (Default: `Dockerfile`) - -**Advantages:** -- Live code updates without rebuilding images -- Smaller image sizes -- Consistent module versions across workflows -- Faster development iteration - -**How it works:** -- Docker image contains only external tools (TruffleHog, Gitleaks) -- Python modules are mounted at runtime from the backend container -- Backend manages code synchronization via shared volumes - -### 2. Self-Contained Approach (`Dockerfile.self-contained`) - -**Advantages:** -- Complete isolation and reproducibility -- No runtime dependencies on backend code -- Can run independently of FuzzForge platform -- Better for CI/CD integration - -**How it works:** -- All required Python modules are copied into the Docker image -- Image is completely self-contained -- Larger image size but fully portable - -## Configuration - -### TruffleHog Configuration - -```json -{ - "trufflehog_config": { - "verify": true, // Verify discovered secrets - "concurrency": 10, // Number of concurrent workers - "max_depth": 10, // Maximum directory depth - "include_detectors": [], // Specific detectors to include - "exclude_detectors": [] // Specific detectors to exclude - } -} -``` - -### Gitleaks Configuration - -```json -{ - "gitleaks_config": { - "scan_mode": "detect", // "detect" or "protect" - "redact": true, // Redact secrets in output - "max_target_megabytes": 100, // Maximum file size (MB) - "no_git": false, // Scan without Git context - "config_file": "", // Custom Gitleaks config - "baseline_file": "" // Baseline file for known findings - } -} -``` - -## Usage Example - -```bash -curl -X POST "http://localhost:8000/workflows/secret_detection_scan/submit" \ - -H "Content-Type: application/json" \ - -d '{ - "target_path": "/path/to/scan", - "volume_mode": "ro", - "parameters": { - "trufflehog_config": { - "verify": true, - "concurrency": 15 - }, - "gitleaks_config": { - "scan_mode": "detect", - "max_target_megabytes": 200 - } - } - }' -``` - -## Output Format - -The workflow generates a SARIF report containing: -- All unique findings from both tools -- Severity levels mapped to standard scale -- File locations and line numbers -- Detailed descriptions and recommendations -- Tool-specific metadata - -## Performance Considerations - -- **TruffleHog**: CPU-intensive with verification enabled -- **Gitleaks**: Memory-intensive for large repositories -- **Recommended Resources**: 512Mi memory, 500m CPU -- **Typical Runtime**: 1-5 minutes for small repos, 10-30 minutes for large ones - -## Security Notes - -- Secrets are redacted in output by default -- Verified secrets are marked with higher severity -- Both tools support custom rules and exclusions -- Consider using baseline files for known false positives \ No newline at end of file diff --git a/backend/toolbox/workflows/comprehensive/secret_detection_scan/metadata.yaml b/backend/toolbox/workflows/comprehensive/secret_detection_scan/metadata.yaml deleted file mode 100644 index 01586e7..0000000 --- a/backend/toolbox/workflows/comprehensive/secret_detection_scan/metadata.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: secret_detection_scan -version: "2.0.0" -description: "Comprehensive secret detection using TruffleHog and Gitleaks" -author: "FuzzForge Team" -category: "comprehensive" -tags: - - "secrets" - - "credentials" - - "detection" - - "trufflehog" - - "gitleaks" - - "comprehensive" - -supported_volume_modes: - - "ro" - - "rw" - -default_volume_mode: "ro" -default_target_path: "/workspace" - -requirements: - tools: - - "trufflehog" - - "gitleaks" - resources: - memory: "512Mi" - cpu: "500m" - timeout: 1800 - -has_docker: true - -default_parameters: - target_path: "/workspace" - volume_mode: "ro" - trufflehog_config: {} - gitleaks_config: {} - reporter_config: {} - -parameters: - type: object - properties: - target_path: - type: string - default: "/workspace" - description: "Path to analyze" - volume_mode: - type: string - enum: ["ro", "rw"] - default: "ro" - description: "Volume mount mode" - trufflehog_config: - type: object - description: "TruffleHog configuration" - properties: - verify: - type: boolean - description: "Verify discovered secrets" - concurrency: - type: integer - description: "Number of concurrent workers" - max_depth: - type: integer - description: "Maximum directory depth to scan" - include_detectors: - type: array - items: - type: string - description: "Specific detectors to include" - exclude_detectors: - type: array - items: - type: string - description: "Specific detectors to exclude" - gitleaks_config: - type: object - description: "Gitleaks configuration" - properties: - scan_mode: - type: string - enum: ["detect", "protect"] - description: "Scan mode" - redact: - type: boolean - description: "Redact secrets in output" - max_target_megabytes: - type: integer - description: "Maximum file size to scan (MB)" - no_git: - type: boolean - description: "Scan files without Git context" - config_file: - type: string - description: "Path to custom configuration file" - baseline_file: - type: string - description: "Path to baseline file" - reporter_config: - type: object - description: "SARIF reporter configuration" - properties: - output_file: - type: string - description: "Output SARIF file name" - include_code_flows: - type: boolean - description: "Include code flow information" - -output_schema: - type: object - properties: - sarif: - type: object - description: "SARIF-formatted security findings" diff --git a/backend/toolbox/workflows/comprehensive/secret_detection_scan/workflow.py b/backend/toolbox/workflows/comprehensive/secret_detection_scan/workflow.py deleted file mode 100644 index f13bbe9..0000000 --- a/backend/toolbox/workflows/comprehensive/secret_detection_scan/workflow.py +++ /dev/null @@ -1,290 +0,0 @@ -""" -Secret Detection Scan Workflow - -This workflow performs comprehensive secret detection using multiple tools: -- TruffleHog: Comprehensive secret detection with verification -- Gitleaks: Git-specific secret scanning -""" -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - - -import sys -import logging -from pathlib import Path -from typing import Dict, Any, List, Optional -from prefect import flow, task -from prefect.artifacts import create_markdown_artifact, create_table_artifact -import asyncio -import json - -# Add modules to path -sys.path.insert(0, '/app') - -# Import modules -from toolbox.modules.secret_detection.trufflehog import TruffleHogModule -from toolbox.modules.secret_detection.gitleaks import GitleaksModule -from toolbox.modules.reporter import SARIFReporter - -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -@task(name="trufflehog_scan") -async def run_trufflehog_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: - """ - Task to run TruffleHog secret detection. - - Args: - workspace: Path to the workspace - config: TruffleHog configuration - - Returns: - TruffleHog results - """ - logger.info("Running TruffleHog secret detection") - module = TruffleHogModule() - result = await module.execute(config, workspace) - logger.info(f"TruffleHog completed: {result.summary.get('total_secrets', 0)} secrets found") - return result.dict() - - -@task(name="gitleaks_scan") -async def run_gitleaks_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: - """ - Task to run Gitleaks secret detection. - - Args: - workspace: Path to the workspace - config: Gitleaks configuration - - Returns: - Gitleaks results - """ - logger.info("Running Gitleaks secret detection") - module = GitleaksModule() - result = await module.execute(config, workspace) - logger.info(f"Gitleaks completed: {result.summary.get('total_leaks', 0)} leaks found") - return result.dict() - - -@task(name="aggregate_findings") -async def aggregate_findings_task( - trufflehog_results: Dict[str, Any], - gitleaks_results: Dict[str, Any], - config: Dict[str, Any], - workspace: Path -) -> Dict[str, Any]: - """ - Task to aggregate findings from all secret detection tools. - - Args: - trufflehog_results: Results from TruffleHog - gitleaks_results: Results from Gitleaks - config: Reporter configuration - workspace: Path to workspace - - Returns: - Aggregated SARIF report - """ - logger.info("Aggregating secret detection findings") - - # Combine all findings - all_findings = [] - - # Add TruffleHog findings - trufflehog_findings = trufflehog_results.get("findings", []) - all_findings.extend(trufflehog_findings) - - # Add Gitleaks findings - gitleaks_findings = gitleaks_results.get("findings", []) - all_findings.extend(gitleaks_findings) - - # Deduplicate findings based on file path and line number - unique_findings = [] - seen_signatures = set() - - for finding in all_findings: - # Create signature for deduplication - signature = ( - finding.get("file_path", ""), - finding.get("line_start", 0), - finding.get("title", "").lower()[:50] # First 50 chars of title - ) - - if signature not in seen_signatures: - seen_signatures.add(signature) - unique_findings.append(finding) - else: - logger.debug(f"Deduplicated finding: {signature}") - - logger.info(f"Aggregated {len(unique_findings)} unique findings from {len(all_findings)} total") - - # Generate SARIF report - reporter = SARIFReporter() - reporter_config = { - **config, - "findings": unique_findings, - "tool_name": "FuzzForge Secret Detection", - "tool_version": "1.0.0", - "tool_description": "Comprehensive secret detection using TruffleHog and Gitleaks" - } - - result = await reporter.execute(reporter_config, workspace) - return result.dict().get("sarif", {}) - - -@flow(name="secret_detection_scan", log_prints=True) -async def main_flow( - target_path: str = "/workspace", - volume_mode: str = "ro", - trufflehog_config: Optional[Dict[str, Any]] = None, - gitleaks_config: Optional[Dict[str, Any]] = None, - reporter_config: Optional[Dict[str, Any]] = None -) -> Dict[str, Any]: - """ - Main secret detection workflow. - - This workflow: - 1. Runs TruffleHog for comprehensive secret detection - 2. Runs Gitleaks for Git-specific secret detection - 3. Aggregates and deduplicates findings - 4. Generates a unified SARIF report - - Args: - target_path: Path to the mounted workspace (default: /workspace) - volume_mode: Volume mount mode (ro/rw) - trufflehog_config: Configuration for TruffleHog - gitleaks_config: Configuration for Gitleaks - reporter_config: Configuration for SARIF reporter - - Returns: - SARIF-formatted findings report - """ - logger.info("Starting comprehensive secret detection workflow") - logger.info(f"Workspace: {target_path}, Mode: {volume_mode}") - - # Set workspace path - workspace = Path(target_path) - - if not workspace.exists(): - logger.error(f"Workspace does not exist: {workspace}") - return { - "error": f"Workspace not found: {workspace}", - "sarif": None - } - - # Default configurations - merge with provided configs to ensure defaults are always applied - default_trufflehog_config = { - "verify": False, - "concurrency": 10, - "max_depth": 10, - "no_git": True # Add no_git for filesystem scanning - } - trufflehog_config = {**default_trufflehog_config, **(trufflehog_config or {})} - - default_gitleaks_config = { - "scan_mode": "detect", - "redact": True, - "max_target_megabytes": 100, - "no_git": True # Critical for non-git directories - } - gitleaks_config = {**default_gitleaks_config, **(gitleaks_config or {})} - - default_reporter_config = { - "include_code_flows": False - } - reporter_config = {**default_reporter_config, **(reporter_config or {})} - - try: - # Run secret detection tools in parallel - logger.info("Phase 1: Running secret detection tools") - - # Create tasks for parallel execution - trufflehog_task_result = run_trufflehog_task(workspace, trufflehog_config) - gitleaks_task_result = run_gitleaks_task(workspace, gitleaks_config) - - # Wait for both to complete - trufflehog_results, gitleaks_results = await asyncio.gather( - trufflehog_task_result, - gitleaks_task_result, - return_exceptions=True - ) - - # Handle any exceptions - if isinstance(trufflehog_results, Exception): - logger.error(f"TruffleHog failed: {trufflehog_results}") - trufflehog_results = {"findings": [], "status": "failed"} - - if isinstance(gitleaks_results, Exception): - logger.error(f"Gitleaks failed: {gitleaks_results}") - gitleaks_results = {"findings": [], "status": "failed"} - - # Aggregate findings - logger.info("Phase 2: Aggregating findings") - sarif_report = await aggregate_findings_task( - trufflehog_results, - gitleaks_results, - reporter_config, - workspace - ) - - # Log summary - if sarif_report and "runs" in sarif_report: - results_count = len(sarif_report["runs"][0].get("results", [])) - logger.info(f"Workflow completed successfully with {results_count} unique secret findings") - - # Log tool-specific stats - trufflehog_count = len(trufflehog_results.get("findings", [])) - gitleaks_count = len(gitleaks_results.get("findings", [])) - logger.info(f"Tool results - TruffleHog: {trufflehog_count}, Gitleaks: {gitleaks_count}") - else: - logger.info("Workflow completed successfully with no findings") - - return sarif_report - - except Exception as e: - logger.error(f"Secret detection workflow failed: {e}") - # Return error in SARIF format - return { - "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", - "version": "2.1.0", - "runs": [ - { - "tool": { - "driver": { - "name": "FuzzForge Secret Detection", - "version": "1.0.0" - } - }, - "results": [], - "invocations": [ - { - "executionSuccessful": False, - "exitCode": 1, - "exitCodeDescription": str(e) - } - ] - } - ] - } - - -if __name__ == "__main__": - # For local testing - import asyncio - - asyncio.run(main_flow( - target_path="/tmp/test", - trufflehog_config={"verify": True, "max_depth": 5}, - gitleaks_config={"scan_mode": "detect"} - )) \ No newline at end of file diff --git a/backend/toolbox/workflows/comprehensive/__init__.py b/backend/toolbox/workflows/gitleaks_detection/__init__.py similarity index 71% rename from backend/toolbox/workflows/comprehensive/__init__.py rename to backend/toolbox/workflows/gitleaks_detection/__init__.py index 83b7d4a..e192e0e 100644 --- a/backend/toolbox/workflows/comprehensive/__init__.py +++ b/backend/toolbox/workflows/gitleaks_detection/__init__.py @@ -1,3 +1,7 @@ +""" +Gitleaks Detection Workflow +""" + # Copyright (c) 2025 FuzzingLabs # # Licensed under the Business Source License 1.1 (BSL). See the LICENSE file @@ -9,4 +13,7 @@ # # Additional attribution and requirements are provided in the NOTICE file. +from .workflow import GitleaksDetectionWorkflow +from .activities import scan_with_gitleaks +__all__ = ["GitleaksDetectionWorkflow", "scan_with_gitleaks"] diff --git a/backend/toolbox/workflows/gitleaks_detection/activities.py b/backend/toolbox/workflows/gitleaks_detection/activities.py new file mode 100644 index 0000000..c7273a3 --- /dev/null +++ b/backend/toolbox/workflows/gitleaks_detection/activities.py @@ -0,0 +1,166 @@ +""" +Gitleaks Detection Workflow Activities +""" + +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +import logging +from pathlib import Path +from typing import Dict, Any + +from temporalio import activity + +try: + from toolbox.modules.secret_detection.gitleaks import GitleaksModule +except ImportError: + try: + from modules.secret_detection.gitleaks import GitleaksModule + except ImportError: + from src.toolbox.modules.secret_detection.gitleaks import GitleaksModule + +logger = logging.getLogger(__name__) + + +@activity.defn(name="scan_with_gitleaks") +async def scan_with_gitleaks(target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Scan code using Gitleaks. + + Args: + target_path: Path to the workspace containing code + config: Gitleaks configuration + + Returns: + Dictionary containing findings and summary + """ + activity.logger.info(f"Starting Gitleaks scan: {target_path}") + activity.logger.info(f"Config: {config}") + + workspace = Path(target_path) + + if not workspace.exists(): + raise FileNotFoundError(f"Workspace not found: {target_path}") + + # Create and execute Gitleaks module + gitleaks = GitleaksModule() + + # Validate configuration + gitleaks.validate_config(config) + + # Execute scan + result = await gitleaks.execute(config, workspace) + + if result.status == "failed": + raise RuntimeError(f"Gitleaks scan failed: {result.error or 'Unknown error'}") + + activity.logger.info( + f"Gitleaks scan completed: {len(result.findings)} findings from " + f"{result.summary.get('files_scanned', 0)} files" + ) + + # Convert ModuleFinding objects to dicts for serialization + findings_dicts = [finding.model_dump() for finding in result.findings] + + return { + "findings": findings_dicts, + "summary": result.summary + } + + +@activity.defn(name="gitleaks_generate_sarif") +async def gitleaks_generate_sarif(findings: list, metadata: Dict[str, Any]) -> Dict[str, Any]: + """ + Generate SARIF report from Gitleaks findings. + + Args: + findings: List of finding dictionaries + metadata: Metadata including tool_name, tool_version, run_id + + Returns: + SARIF report dictionary + """ + activity.logger.info(f"Generating SARIF report from {len(findings)} findings") + + # Debug: Check if first finding has line_start + if findings: + first_finding = findings[0] + activity.logger.info(f"First finding keys: {list(first_finding.keys())}") + activity.logger.info(f"line_start value: {first_finding.get('line_start')}") + + # Basic SARIF 2.1.0 structure + sarif_report = { + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": metadata.get("tool_name", "gitleaks"), + "version": metadata.get("tool_version", "8.18.0"), + "informationUri": "https://github.com/gitleaks/gitleaks" + } + }, + "results": [] + } + ] + } + + # Convert findings to SARIF results + for finding in findings: + sarif_result = { + "ruleId": finding.get("metadata", {}).get("rule_id", "unknown"), + "level": _severity_to_sarif_level(finding.get("severity", "warning")), + "message": { + "text": finding.get("title", "Secret leak detected") + }, + "locations": [] + } + + # Add description if present + if finding.get("description"): + sarif_result["message"]["markdown"] = finding["description"] + + # Add location if file path is present + if finding.get("file_path"): + location = { + "physicalLocation": { + "artifactLocation": { + "uri": finding["file_path"] + } + } + } + + # Add region if line number is present + if finding.get("line_start"): + location["physicalLocation"]["region"] = { + "startLine": finding["line_start"] + } + + sarif_result["locations"].append(location) + + sarif_report["runs"][0]["results"].append(sarif_result) + + activity.logger.info(f"Generated SARIF report with {len(sarif_report['runs'][0]['results'])} results") + + return sarif_report + + +def _severity_to_sarif_level(severity: str) -> str: + """Convert severity to SARIF level""" + severity_map = { + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note" + } + return severity_map.get(severity.lower(), "warning") diff --git a/backend/toolbox/workflows/gitleaks_detection/metadata.yaml b/backend/toolbox/workflows/gitleaks_detection/metadata.yaml new file mode 100644 index 0000000..d2c343c --- /dev/null +++ b/backend/toolbox/workflows/gitleaks_detection/metadata.yaml @@ -0,0 +1,42 @@ +name: gitleaks_detection +version: "1.0.0" +vertical: secrets +description: "Detect secrets and credentials using Gitleaks" +author: "FuzzForge Team" +tags: + - "secrets" + - "gitleaks" + - "git" + - "leak-detection" + +workspace_isolation: "shared" + +parameters: + type: object + properties: + scan_mode: + type: string + enum: ["detect", "protect"] + default: "detect" + description: "Scan mode: detect (entire repo history) or protect (staged changes)" + + redact: + type: boolean + default: true + description: "Redact secrets in output" + + no_git: + type: boolean + default: false + description: "Scan files without Git context" + +default_parameters: + scan_mode: "detect" + redact: true + no_git: false + +required_modules: + - "gitleaks" + +supported_volume_modes: + - "ro" diff --git a/backend/toolbox/workflows/gitleaks_detection/workflow.py b/backend/toolbox/workflows/gitleaks_detection/workflow.py new file mode 100644 index 0000000..4960e16 --- /dev/null +++ b/backend/toolbox/workflows/gitleaks_detection/workflow.py @@ -0,0 +1,187 @@ +""" +Gitleaks Detection Workflow - Temporal Version + +Scans code for secrets and credentials using Gitleaks. +""" + +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +from datetime import timedelta +from typing import Dict, Any + +from temporalio import workflow +from temporalio.common import RetryPolicy + +# Import for type hints (will be executed by worker) +with workflow.unsafe.imports_passed_through(): + import logging + +logger = logging.getLogger(__name__) + + +@workflow.defn +class GitleaksDetectionWorkflow: + """ + Scan code for secrets using Gitleaks. + + User workflow: + 1. User runs: ff workflow run gitleaks_detection . + 2. CLI uploads project to MinIO + 3. Worker downloads project + 4. Worker runs Gitleaks + 5. Secrets reported as findings in SARIF format + """ + + @workflow.run + async def run( + self, + target_id: str, # MinIO UUID of uploaded user code + scan_mode: str = "detect", + redact: bool = True, + no_git: bool = True + ) -> Dict[str, Any]: + """ + Main workflow execution. + + Args: + target_id: UUID of the uploaded target in MinIO + scan_mode: Scan mode ('detect' or 'protect') + redact: Redact secrets in output + no_git: Scan files without Git context + + Returns: + Dictionary containing findings and summary + """ + workflow_id = workflow.info().workflow_id + + workflow.logger.info( + f"Starting GitleaksDetectionWorkflow " + f"(workflow_id={workflow_id}, target_id={target_id}, scan_mode={scan_mode})" + ) + + results = { + "workflow_id": workflow_id, + "target_id": target_id, + "status": "running", + "steps": [], + "findings": [] + } + + try: + # Get run ID for workspace isolation + run_id = workflow.info().run_id + + # Step 1: Download user's project from MinIO + workflow.logger.info("Step 1: Downloading user code from MinIO") + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "shared"], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["steps"].append({ + "step": "download", + "status": "success", + "target_path": target_path + }) + workflow.logger.info(f"āœ“ Target downloaded to: {target_path}") + + # Step 2: Run Gitleaks + workflow.logger.info("Step 2: Scanning with Gitleaks") + + scan_config = { + "scan_mode": scan_mode, + "redact": redact, + "no_git": no_git + } + + scan_results = await workflow.execute_activity( + "scan_with_gitleaks", + args=[target_path, scan_config], + start_to_close_timeout=timedelta(minutes=10), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=2 + ) + ) + + results["steps"].append({ + "step": "gitleaks_scan", + "status": "success", + "leaks_found": scan_results.get("summary", {}).get("total_leaks", 0) + }) + workflow.logger.info( + f"āœ“ Gitleaks scan completed: " + f"{scan_results.get('summary', {}).get('total_leaks', 0)} leaks found" + ) + + # Step 3: Generate SARIF report + workflow.logger.info("Step 3: Generating SARIF report") + sarif_report = await workflow.execute_activity( + "gitleaks_generate_sarif", + args=[scan_results.get("findings", []), {"tool_name": "gitleaks", "tool_version": "8.18.0"}], + start_to_close_timeout=timedelta(minutes=2) + ) + + # Step 4: Upload results to MinIO + workflow.logger.info("Step 4: Uploading results") + try: + results_url = await workflow.execute_activity( + "upload_results", + args=[workflow_id, scan_results, "json"], + start_to_close_timeout=timedelta(minutes=2) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + except Exception as e: + workflow.logger.warning(f"Failed to upload results: {e}") + results["results_url"] = None + + # Step 5: Cleanup cache + workflow.logger.info("Step 5: Cleaning up cache") + try: + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "shared"], + start_to_close_timeout=timedelta(minutes=1) + ) + workflow.logger.info("āœ“ Cache cleaned up") + except Exception as e: + workflow.logger.warning(f"Cache cleanup failed: {e}") + + # Mark workflow as successful + results["status"] = "success" + results["findings"] = scan_results.get("findings", []) + results["summary"] = scan_results.get("summary", {}) + results["sarif"] = sarif_report or {} + workflow.logger.info( + f"āœ“ Workflow completed successfully: {workflow_id} " + f"({results['summary'].get('total_leaks', 0)} leaks found)" + ) + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + results["steps"].append({ + "step": "error", + "status": "failed", + "error": str(e) + }) + raise diff --git a/backend/toolbox/workflows/comprehensive/secret_detection_scan/__init__.py b/backend/toolbox/workflows/llm_analysis/__init__.py similarity index 74% rename from backend/toolbox/workflows/comprehensive/secret_detection_scan/__init__.py rename to backend/toolbox/workflows/llm_analysis/__init__.py index bb5379d..028946c 100644 --- a/backend/toolbox/workflows/comprehensive/secret_detection_scan/__init__.py +++ b/backend/toolbox/workflows/llm_analysis/__init__.py @@ -1,9 +1,7 @@ """ -Secret Detection Scan Workflow +LLM Analysis Workflow +""" -This package contains the comprehensive secret detection workflow that combines -multiple secret detection tools for thorough analysis. -""" # Copyright (c) 2025 FuzzingLabs # # Licensed under the Business Source License 1.1 (BSL). See the LICENSE file @@ -15,3 +13,7 @@ multiple secret detection tools for thorough analysis. # # Additional attribution and requirements are provided in the NOTICE file. +from .workflow import LlmAnalysisWorkflow +from .activities import analyze_with_llm + +__all__ = ["LlmAnalysisWorkflow", "analyze_with_llm"] diff --git a/backend/toolbox/workflows/llm_analysis/activities.py b/backend/toolbox/workflows/llm_analysis/activities.py new file mode 100644 index 0000000..cb47599 --- /dev/null +++ b/backend/toolbox/workflows/llm_analysis/activities.py @@ -0,0 +1,162 @@ +""" +LLM Analysis Workflow Activities +""" + +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +import logging +from pathlib import Path +from typing import Dict, Any + +from temporalio import activity + +try: + from toolbox.modules.analyzer.llm_analyzer import LLMAnalyzer +except ImportError: + try: + from modules.analyzer.llm_analyzer import LLMAnalyzer + except ImportError: + from src.toolbox.modules.analyzer.llm_analyzer import LLMAnalyzer + +logger = logging.getLogger(__name__) + + +@activity.defn(name="llm_generate_sarif") +async def llm_generate_sarif(findings: list, metadata: Dict[str, Any]) -> Dict[str, Any]: + """ + Generate SARIF report from LLM findings. + + Args: + findings: List of finding dictionaries + metadata: Metadata including tool_name, tool_version, run_id + + Returns: + SARIF report dictionary + """ + activity.logger.info(f"Generating SARIF report from {len(findings)} findings") + + # Basic SARIF 2.1.0 structure + sarif_report = { + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": metadata.get("tool_name", "llm-analyzer"), + "version": metadata.get("tool_version", "1.0.0"), + "informationUri": "https://github.com/FuzzingLabs/fuzzforge_ai" + } + }, + "results": [] + } + ] + } + + # Convert findings to SARIF results + for finding in findings: + sarif_result = { + "ruleId": finding.get("id", "unknown"), + "level": _severity_to_sarif_level(finding.get("severity", "warning")), + "message": { + "text": finding.get("title", "Security issue detected") + }, + "locations": [] + } + + # Add description if present + if finding.get("description"): + sarif_result["message"]["markdown"] = finding["description"] + + # Add location if file path is present + if finding.get("file_path"): + location = { + "physicalLocation": { + "artifactLocation": { + "uri": finding["file_path"] + } + } + } + + # Add region if line number is present + if finding.get("line_start"): + location["physicalLocation"]["region"] = { + "startLine": finding["line_start"] + } + if finding.get("line_end"): + location["physicalLocation"]["region"]["endLine"] = finding["line_end"] + + sarif_result["locations"].append(location) + + sarif_report["runs"][0]["results"].append(sarif_result) + + activity.logger.info(f"Generated SARIF report with {len(sarif_report['runs'][0]['results'])} results") + + return sarif_report + + +def _severity_to_sarif_level(severity: str) -> str: + """Convert severity to SARIF level""" + severity_map = { + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note" + } + return severity_map.get(severity.lower(), "warning") + + +@activity.defn(name="analyze_with_llm") +async def analyze_with_llm(target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Analyze code using LLM. + + Args: + target_path: Path to the workspace containing code + config: LLM analyzer configuration + + Returns: + Dictionary containing findings and summary + """ + activity.logger.info(f"Starting LLM analysis: {target_path}") + activity.logger.info(f"Config: {config}") + + workspace = Path(target_path) + + if not workspace.exists(): + raise FileNotFoundError(f"Workspace not found: {target_path}") + + # Create and execute LLM analyzer + analyzer = LLMAnalyzer() + + # Validate configuration + analyzer.validate_config(config) + + # Execute analysis + result = await analyzer.execute(config, workspace) + + if result.status == "failed": + raise RuntimeError(f"LLM analysis failed: {result.error or 'Unknown error'}") + + activity.logger.info( + f"LLM analysis completed: {len(result.findings)} findings from " + f"{result.summary.get('files_analyzed', 0)} files" + ) + + # Convert ModuleFinding objects to dicts for serialization + findings_dicts = [finding.model_dump() for finding in result.findings] + + return { + "findings": findings_dicts, + "summary": result.summary + } diff --git a/backend/toolbox/workflows/llm_analysis/metadata.yaml b/backend/toolbox/workflows/llm_analysis/metadata.yaml new file mode 100644 index 0000000..0a388bf --- /dev/null +++ b/backend/toolbox/workflows/llm_analysis/metadata.yaml @@ -0,0 +1,64 @@ +name: llm_analysis +version: "1.0.0" +vertical: python +description: "Uses AI/LLM to analyze code for security vulnerabilities and code quality issues" +author: "FuzzForge Team" +tags: + - "llm" + - "ai" + - "security" + - "static-analysis" + - "code-quality" + +# Workspace isolation mode +workspace_isolation: "shared" + +default_parameters: + agent_url: "http://fuzzforge-task-agent:8000/a2a/litellm_agent" + llm_model: "gpt-5-mini" + llm_provider: "openai" + max_files: 5 + +parameters: + type: object + properties: + agent_url: + type: string + description: "A2A agent endpoint URL" + llm_model: + type: string + description: "LLM model to use (e.g., gpt-4o-mini, claude-3-5-sonnet)" + llm_provider: + type: string + description: "LLM provider (openai, anthropic, etc.)" + file_patterns: + type: array + items: + type: string + description: "File patterns to analyze (e.g., ['*.py', '*.js'])" + max_files: + type: integer + description: "Maximum number of files to analyze" + max_file_size: + type: integer + description: "Maximum file size in bytes" + timeout: + type: integer + description: "Timeout per file in seconds" + +output_schema: + type: object + properties: + sarif: + type: object + description: "SARIF-formatted security findings from LLM" + summary: + type: object + description: "Analysis summary" + properties: + files_analyzed: + type: integer + total_findings: + type: integer + model_used: + type: string diff --git a/backend/toolbox/workflows/llm_analysis/workflow.py b/backend/toolbox/workflows/llm_analysis/workflow.py new file mode 100644 index 0000000..136e844 --- /dev/null +++ b/backend/toolbox/workflows/llm_analysis/workflow.py @@ -0,0 +1,236 @@ +""" +LLM Analysis Workflow - Temporal Version + +Uses AI/LLM to analyze code for security issues. +""" + +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +from datetime import timedelta +from typing import Dict, Any, Optional + +from temporalio import workflow +from temporalio.common import RetryPolicy + +# Import for type hints (will be executed by worker) +with workflow.unsafe.imports_passed_through(): + import logging + +logger = logging.getLogger(__name__) + + +@workflow.defn +class LlmAnalysisWorkflow: + """ + Analyze code using AI/LLM for security vulnerabilities. + + User workflow: + 1. User runs: ff workflow run llm_analysis . + 2. CLI uploads project to MinIO + 3. Worker downloads project + 4. Worker calls LLM analyzer module + 5. LLM analyzes code files and reports findings + 6. Results returned in SARIF format + """ + + @workflow.run + async def run( + self, + target_id: str, # MinIO UUID of uploaded user code + agent_url: Optional[str] = None, + llm_model: Optional[str] = None, + llm_provider: Optional[str] = None, + file_patterns: Optional[list] = None, + max_files: Optional[int] = None, + max_file_size: Optional[int] = None, + timeout: Optional[int] = None + ) -> Dict[str, Any]: + """ + Main workflow execution. + + Args: + target_id: UUID of the uploaded target in MinIO + agent_url: A2A agent endpoint URL + llm_model: LLM model to use + llm_provider: LLM provider + file_patterns: File patterns to analyze + max_files: Maximum number of files to analyze + max_file_size: Maximum file size in bytes + timeout: Timeout per file in seconds + + Returns: + Dictionary containing findings and summary + """ + workflow_id = workflow.info().workflow_id + + workflow.logger.info( + f"Starting LLMAnalysisWorkflow " + f"(workflow_id={workflow_id}, target_id={target_id}, model={llm_model})" + ) + + results = { + "workflow_id": workflow_id, + "target_id": target_id, + "status": "running", + "steps": [], + "findings": [] + } + + try: + # Get run ID for workspace isolation + run_id = workflow.info().run_id + + # Step 1: Download user's project from MinIO + workflow.logger.info("Step 1: Downloading user code from MinIO") + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "shared"], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["steps"].append({ + "step": "download", + "status": "success", + "target_path": target_path + }) + workflow.logger.info(f"āœ“ Target downloaded to: {target_path}") + + # Step 2: Run LLM analysis + workflow.logger.info("Step 2: Analyzing code with LLM") + + # Build analyzer config + analyzer_config = {} + if agent_url: + analyzer_config["agent_url"] = agent_url + if llm_model: + analyzer_config["llm_model"] = llm_model + if llm_provider: + analyzer_config["llm_provider"] = llm_provider + if file_patterns: + analyzer_config["file_patterns"] = file_patterns + if max_files is not None: + analyzer_config["max_files"] = max_files + if max_file_size is not None: + analyzer_config["max_file_size"] = max_file_size + if timeout is not None: + analyzer_config["timeout"] = timeout + + analysis_results = await workflow.execute_activity( + "analyze_with_llm", + args=[target_path, analyzer_config], + start_to_close_timeout=timedelta(minutes=30), # LLM calls can be slow + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=5), + maximum_interval=timedelta(minutes=1), + maximum_attempts=2 + ) + ) + + findings = analysis_results.get("findings", []) + summary = analysis_results.get("summary", {}) + + results["steps"].append({ + "step": "llm_analysis", + "status": "success", + "files_analyzed": summary.get("files_analyzed", 0), + "findings_count": len(findings) + }) + + workflow.logger.info( + f"āœ“ LLM analysis completed: " + f"{summary.get('files_analyzed', 0)} files, " + f"{len(findings)} findings" + ) + + # Step 3: Generate SARIF report + workflow.logger.info("Step 3: Generating SARIF report") + + sarif_report = await workflow.execute_activity( + "llm_generate_sarif", + args=[findings, { + "tool_name": "llm-analyzer", + "tool_version": "1.0.0", + "run_id": run_id + }], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + + results["steps"].append({ + "step": "sarif_generation", + "status": "success", + "results_count": len(sarif_report.get("runs", [{}])[0].get("results", [])) + }) + + workflow.logger.info( + f"āœ“ SARIF report generated: " + f"{len(sarif_report.get('runs', [{}])[0].get('results', []))} results" + ) + + # Step 4: Upload results to MinIO + workflow.logger.info("Step 4: Uploading results to MinIO") + + # Upload SARIF report + if sarif_report: + results_url = await workflow.execute_activity( + "upload_results", + args=[run_id, sarif_report], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + + # Step 5: Cleanup cache + workflow.logger.info("Step 5: Cleaning up cache") + await workflow.execute_activity( + "cleanup_cache", + args=[target_id], + start_to_close_timeout=timedelta(minutes=2), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=10), + maximum_attempts=2 + ) + ) + workflow.logger.info("āœ“ Cache cleaned up") + + # Mark workflow as successful + results["status"] = "success" + results["sarif"] = sarif_report + results["summary"] = summary + results["findings"] = findings + + workflow.logger.info( + f"āœ… LLMAnalysisWorkflow completed successfully: " + f"{len(findings)} findings" + ) + + except Exception as e: + workflow.logger.error(f"āŒ Workflow failed: {e}") + results["status"] = "failed" + results["error"] = str(e) + raise + + return results diff --git a/backend/toolbox/workflows/llm_secret_detection/__init__.py b/backend/toolbox/workflows/llm_secret_detection/__init__.py new file mode 100644 index 0000000..81148a7 --- /dev/null +++ b/backend/toolbox/workflows/llm_secret_detection/__init__.py @@ -0,0 +1,6 @@ +"""LLM Secret Detection Workflow""" + +from .workflow import LlmSecretDetectionWorkflow +from .activities import scan_with_llm + +__all__ = ["LlmSecretDetectionWorkflow", "scan_with_llm"] diff --git a/backend/toolbox/workflows/llm_secret_detection/activities.py b/backend/toolbox/workflows/llm_secret_detection/activities.py new file mode 100644 index 0000000..c16691f --- /dev/null +++ b/backend/toolbox/workflows/llm_secret_detection/activities.py @@ -0,0 +1,112 @@ +"""LLM Secret Detection Workflow Activities""" + +from pathlib import Path +from typing import Dict, Any +from temporalio import activity + +try: + from toolbox.modules.secret_detection.llm_secret_detector import LLMSecretDetectorModule +except ImportError: + from modules.secret_detection.llm_secret_detector import LLMSecretDetectorModule + +@activity.defn(name="scan_with_llm") +async def scan_with_llm(target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: + """Scan code using LLM.""" + activity.logger.info(f"Starting LLM secret detection: {target_path}") + workspace = Path(target_path) + + llm_detector = LLMSecretDetectorModule() + llm_detector.validate_config(config) + result = await llm_detector.execute(config, workspace) + + if result.status == "failed": + raise RuntimeError(f"LLM detection failed: {result.error}") + + findings_dicts = [finding.model_dump() for finding in result.findings] + return {"findings": findings_dicts, "summary": result.summary} + + +@activity.defn(name="llm_secret_generate_sarif") +async def llm_secret_generate_sarif(findings: list, metadata: Dict[str, Any]) -> Dict[str, Any]: + """ + Generate SARIF report from LLM secret detection findings. + + Args: + findings: List of finding dictionaries from LLM secret detector + metadata: Metadata including tool_name, tool_version + + Returns: + SARIF 2.1.0 report dictionary + """ + activity.logger.info(f"Generating SARIF report from {len(findings)} findings") + + # Basic SARIF 2.1.0 structure + sarif_report = { + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": metadata.get("tool_name", "llm-secret-detector"), + "version": metadata.get("tool_version", "1.0.0"), + "informationUri": "https://github.com/FuzzingLabs/fuzzforge_ai" + } + }, + "results": [] + } + ] + } + + # Convert findings to SARIF results + for finding in findings: + sarif_result = { + "ruleId": finding.get("id", finding.get("metadata", {}).get("secret_type", "unknown-secret")), + "level": _severity_to_sarif_level(finding.get("severity", "warning")), + "message": { + "text": finding.get("title", "Secret detected by LLM") + }, + "locations": [] + } + + # Add description if present + if finding.get("description"): + sarif_result["message"]["markdown"] = finding["description"] + + # Add location if file path is present + if finding.get("file_path"): + location = { + "physicalLocation": { + "artifactLocation": { + "uri": finding["file_path"] + } + } + } + + # Add region if line number is present + if finding.get("line_start"): + location["physicalLocation"]["region"] = { + "startLine": finding["line_start"] + } + if finding.get("line_end"): + location["physicalLocation"]["region"]["endLine"] = finding["line_end"] + + sarif_result["locations"].append(location) + + sarif_report["runs"][0]["results"].append(sarif_result) + + activity.logger.info(f"Generated SARIF report with {len(sarif_report['runs'][0]['results'])} results") + + return sarif_report + + +def _severity_to_sarif_level(severity: str) -> str: + """Convert severity to SARIF level""" + severity_map = { + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note" + } + return severity_map.get(severity.lower(), "warning") diff --git a/backend/toolbox/workflows/llm_secret_detection/metadata.yaml b/backend/toolbox/workflows/llm_secret_detection/metadata.yaml new file mode 100644 index 0000000..91b9c2c --- /dev/null +++ b/backend/toolbox/workflows/llm_secret_detection/metadata.yaml @@ -0,0 +1,43 @@ +name: llm_secret_detection +version: "1.0.0" +vertical: secrets +description: "AI-powered secret detection using LLM semantic analysis" +author: "FuzzForge Team" +tags: + - "secrets" + - "llm" + - "ai" + - "semantic" + +workspace_isolation: "shared" + +parameters: + type: object + properties: + agent_url: + type: string + default: "http://fuzzforge-task-agent:8000/a2a/litellm_agent" + + llm_model: + type: string + default: "gpt-4o-mini" + + llm_provider: + type: string + default: "openai" + + max_files: + type: integer + default: 20 + +default_parameters: + agent_url: "http://fuzzforge-task-agent:8000/a2a/litellm_agent" + llm_model: "gpt-4o-mini" + llm_provider: "openai" + max_files: 20 + +required_modules: + - "llm_secret_detector" + +supported_volume_modes: + - "ro" diff --git a/backend/toolbox/workflows/llm_secret_detection/workflow.py b/backend/toolbox/workflows/llm_secret_detection/workflow.py new file mode 100644 index 0000000..4f693d0 --- /dev/null +++ b/backend/toolbox/workflows/llm_secret_detection/workflow.py @@ -0,0 +1,156 @@ +"""LLM Secret Detection Workflow""" + +from datetime import timedelta +from typing import Dict, Any, Optional +from temporalio import workflow +from temporalio.common import RetryPolicy + +@workflow.defn +class LlmSecretDetectionWorkflow: + """Scan code for secrets using LLM AI.""" + + @workflow.run + async def run( + self, + target_id: str, + agent_url: Optional[str] = None, + llm_model: Optional[str] = None, + llm_provider: Optional[str] = None, + max_files: Optional[int] = None, + timeout: Optional[int] = None, + file_patterns: Optional[list] = None + ) -> Dict[str, Any]: + workflow_id = workflow.info().workflow_id + run_id = workflow.info().run_id + + workflow.logger.info( + f"Starting LLM Secret Detection Workflow " + f"(workflow_id={workflow_id}, target_id={target_id}, model={llm_model})" + ) + + results = { + "workflow_id": workflow_id, + "target_id": target_id, + "status": "running", + "steps": [], + "findings": [] + } + + try: + # Step 1: Download target from MinIO + workflow.logger.info("Step 1: Downloading target from MinIO") + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "shared"], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["steps"].append({ + "step": "download", + "status": "success", + "target_path": target_path + }) + workflow.logger.info(f"āœ“ Target downloaded to: {target_path}") + + # Step 2: Scan with LLM + workflow.logger.info("Step 2: Scanning with LLM") + config = {} + if agent_url: + config["agent_url"] = agent_url + if llm_model: + config["llm_model"] = llm_model + if llm_provider: + config["llm_provider"] = llm_provider + if max_files: + config["max_files"] = max_files + if timeout: + config["timeout"] = timeout + if file_patterns: + config["file_patterns"] = file_patterns + + scan_results = await workflow.execute_activity( + "scan_with_llm", + args=[target_path, config], + start_to_close_timeout=timedelta(minutes=30), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=2 + ) + ) + + findings_count = len(scan_results.get("findings", [])) + results["steps"].append({ + "step": "llm_scan", + "status": "success", + "secrets_found": findings_count + }) + workflow.logger.info(f"āœ“ LLM scan completed: {findings_count} secrets found") + + # Step 3: Generate SARIF report + workflow.logger.info("Step 3: Generating SARIF report") + sarif_report = await workflow.execute_activity( + "llm_generate_sarif", # Use shared LLM SARIF activity + args=[ + scan_results.get("findings", []), + { + "tool_name": f"llm-secret-detector ({llm_model or 'gpt-4o-mini'})", + "tool_version": "1.0.0" + } + ], + start_to_close_timeout=timedelta(minutes=2) + ) + workflow.logger.info("āœ“ SARIF report generated") + + # Step 4: Upload results to MinIO + workflow.logger.info("Step 4: Uploading results") + try: + results_url = await workflow.execute_activity( + "upload_results", + args=[workflow_id, scan_results, "json"], + start_to_close_timeout=timedelta(minutes=2) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + except Exception as e: + workflow.logger.warning(f"Failed to upload results: {e}") + results["results_url"] = None + + # Step 5: Cleanup cache + workflow.logger.info("Step 5: Cleaning up cache") + try: + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "shared"], + start_to_close_timeout=timedelta(minutes=1) + ) + workflow.logger.info("āœ“ Cache cleaned up") + except Exception as e: + workflow.logger.warning(f"Cache cleanup failed: {e}") + + # Mark workflow as successful + results["status"] = "success" + results["findings"] = scan_results.get("findings", []) + results["summary"] = scan_results.get("summary", {}) + results["sarif"] = sarif_report or {} + workflow.logger.info( + f"āœ“ Workflow completed successfully: {workflow_id} " + f"({findings_count} secrets found)" + ) + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + results["steps"].append({ + "step": "error", + "status": "failed", + "error": str(e) + }) + raise diff --git a/backend/toolbox/workflows/ossfuzz_campaign/metadata.yaml b/backend/toolbox/workflows/ossfuzz_campaign/metadata.yaml new file mode 100644 index 0000000..fbc1d51 --- /dev/null +++ b/backend/toolbox/workflows/ossfuzz_campaign/metadata.yaml @@ -0,0 +1,113 @@ +name: ossfuzz_campaign +version: "1.0.0" +vertical: ossfuzz +description: "Generic OSS-Fuzz fuzzing campaign. Automatically reads project configuration from OSS-Fuzz repo and runs fuzzing using Google's infrastructure." +author: "FuzzForge Team" +tags: + - "fuzzing" + - "oss-fuzz" + - "libfuzzer" + - "afl" + - "honggfuzz" + - "memory-safety" + - "security" + +# Workspace isolation mode +# OSS-Fuzz campaigns use isolated mode for safe concurrent campaigns +workspace_isolation: "isolated" + +default_parameters: + project_name: null + campaign_duration_hours: 1 + override_engine: null + override_sanitizer: null + max_iterations: null + +parameters: + type: object + required: + - project_name + properties: + project_name: + type: string + description: "OSS-Fuzz project name (e.g., 'curl', 'sqlite3', 'libxml2')" + examples: + - "curl" + - "sqlite3" + - "libxml2" + - "openssl" + - "zlib" + + campaign_duration_hours: + type: integer + default: 1 + minimum: 1 + maximum: 168 # 1 week max + description: "How many hours to run the fuzzing campaign" + + override_engine: + type: string + enum: ["libfuzzer", "afl", "honggfuzz"] + description: "Override fuzzing engine from project.yaml (optional)" + + override_sanitizer: + type: string + enum: ["address", "memory", "undefined", "dataflow"] + description: "Override sanitizer from project.yaml (optional)" + + max_iterations: + type: integer + minimum: 1000 + description: "Optional limit on fuzzing iterations (optional)" + +output_schema: + type: object + properties: + project_name: + type: string + description: "OSS-Fuzz project that was fuzzed" + + summary: + type: object + description: "Campaign execution summary" + properties: + total_executions: + type: integer + crashes_found: + type: integer + unique_crashes: + type: integer + duration_hours: + type: number + engine_used: + type: string + sanitizer_used: + type: string + + crashes: + type: array + description: "List of crash file paths" + items: + type: string + + sarif: + type: object + description: "SARIF-formatted crash reports (future)" + +examples: + - name: "Fuzz curl for 1 hour" + parameters: + project_name: "curl" + campaign_duration_hours: 1 + + - name: "Fuzz sqlite3 with AFL" + parameters: + project_name: "sqlite3" + campaign_duration_hours: 2 + override_engine: "afl" + + - name: "Fuzz libxml2 with memory sanitizer" + parameters: + project_name: "libxml2" + campaign_duration_hours: 6 + override_sanitizer: "memory" diff --git a/backend/toolbox/workflows/ossfuzz_campaign/workflow.py b/backend/toolbox/workflows/ossfuzz_campaign/workflow.py new file mode 100644 index 0000000..7b735dd --- /dev/null +++ b/backend/toolbox/workflows/ossfuzz_campaign/workflow.py @@ -0,0 +1,219 @@ +""" +OSS-Fuzz Campaign Workflow - Temporal Version + +Generic workflow for running OSS-Fuzz campaigns using Google's infrastructure. +Automatically reads project configuration from OSS-Fuzz project.yaml files. +""" + +import asyncio +from datetime import timedelta +from typing import Dict, Any, Optional + +from temporalio import workflow +from temporalio.common import RetryPolicy + +# Import for type hints (will be executed by worker) +with workflow.unsafe.imports_passed_through(): + import logging + +logger = logging.getLogger(__name__) + + +@workflow.defn +class OssfuzzCampaignWorkflow: + """ + Generic OSS-Fuzz fuzzing campaign workflow. + + User workflow: + 1. User runs: ff workflow run ossfuzz_campaign . project_name=curl + 2. Worker loads project config from OSS-Fuzz repo + 3. Worker builds project using OSS-Fuzz's build system + 4. Worker runs fuzzing with engines from project.yaml + 5. Crashes and corpus reported as findings + """ + + @workflow.run + async def run( + self, + target_id: str, # Required by FuzzForge (not used, OSS-Fuzz downloads from Google) + project_name: str, # Required: OSS-Fuzz project name (e.g., "curl", "sqlite3") + campaign_duration_hours: int = 1, + override_engine: Optional[str] = None, # Override engine from project.yaml + override_sanitizer: Optional[str] = None, # Override sanitizer from project.yaml + max_iterations: Optional[int] = None # Optional: limit fuzzing iterations + ) -> Dict[str, Any]: + """ + Main workflow execution. + + Args: + target_id: UUID of uploaded target (not used, required by FuzzForge) + project_name: Name of OSS-Fuzz project (e.g., "curl", "sqlite3", "libxml2") + campaign_duration_hours: How many hours to fuzz (default: 1) + override_engine: Override fuzzing engine from project.yaml + override_sanitizer: Override sanitizer from project.yaml + max_iterations: Optional limit on fuzzing iterations + + Returns: + Dictionary containing crashes, stats, and SARIF report + """ + workflow_id = workflow.info().workflow_id + + workflow.logger.info( + f"Starting OSS-Fuzz Campaign for project '{project_name}' " + f"(workflow_id={workflow_id}, duration={campaign_duration_hours}h)" + ) + + results = { + "workflow_id": workflow_id, + "project_name": project_name, + "status": "running", + "steps": [] + } + + try: + # Step 1: Load OSS-Fuzz project configuration + workflow.logger.info(f"Step 1: Loading project config for '{project_name}'") + project_config = await workflow.execute_activity( + "load_ossfuzz_project", + args=[project_name], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + + results["steps"].append({ + "step": "load_config", + "status": "success", + "language": project_config.get("language"), + "engines": project_config.get("fuzzing_engines", []), + "sanitizers": project_config.get("sanitizers", []) + }) + + workflow.logger.info( + f"āœ“ Loaded config: language={project_config.get('language')}, " + f"engines={project_config.get('fuzzing_engines')}" + ) + + # Step 2: Build project using OSS-Fuzz infrastructure + workflow.logger.info(f"Step 2: Building project '{project_name}'") + + build_result = await workflow.execute_activity( + "build_ossfuzz_project", + args=[ + project_name, + project_config, + override_sanitizer, + override_engine + ], + start_to_close_timeout=timedelta(minutes=30), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=2 + ) + ) + + results["steps"].append({ + "step": "build_project", + "status": "success", + "fuzz_targets": len(build_result.get("fuzz_targets", [])), + "sanitizer": build_result.get("sanitizer_used"), + "engine": build_result.get("engine_used") + }) + + workflow.logger.info( + f"āœ“ Build completed: {len(build_result.get('fuzz_targets', []))} fuzz targets found" + ) + + if not build_result.get("fuzz_targets"): + raise Exception(f"No fuzz targets found for project {project_name}") + + # Step 3: Run fuzzing on discovered targets + workflow.logger.info(f"Step 3: Fuzzing {len(build_result['fuzz_targets'])} targets") + + # Determine which engine to use + engine_to_use = override_engine if override_engine else build_result["engine_used"] + duration_seconds = campaign_duration_hours * 3600 + + # Fuzz each target (in parallel if multiple targets) + fuzz_futures = [] + for target_path in build_result["fuzz_targets"]: + future = workflow.execute_activity( + "fuzz_target", + args=[target_path, engine_to_use, duration_seconds, None, None], + start_to_close_timeout=timedelta(seconds=duration_seconds + 300), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=1 # Fuzzing shouldn't retry + ) + ) + fuzz_futures.append(future) + + # Wait for all fuzzing to complete + fuzz_results = await asyncio.gather(*fuzz_futures, return_exceptions=True) + + # Aggregate results + total_execs = 0 + total_crashes = 0 + all_crashes = [] + + for i, result in enumerate(fuzz_results): + if isinstance(result, Exception): + workflow.logger.error(f"Fuzzing failed for target {i}: {result}") + continue + + total_execs += result.get("total_executions", 0) + total_crashes += result.get("crashes", 0) + all_crashes.extend(result.get("crash_files", [])) + + results["steps"].append({ + "step": "fuzzing", + "status": "success", + "total_executions": total_execs, + "crashes_found": total_crashes, + "targets_fuzzed": len(build_result["fuzz_targets"]) + }) + + workflow.logger.info( + f"āœ“ Fuzzing completed: {total_execs} executions, {total_crashes} crashes" + ) + + # Step 4: Generate SARIF report + workflow.logger.info("Step 4: Generating SARIF report") + + # TODO: Implement crash minimization and SARIF generation + # For now, return raw results + + results["status"] = "success" + results["summary"] = { + "project": project_name, + "total_executions": total_execs, + "crashes_found": total_crashes, + "unique_crashes": len(set(all_crashes)), + "duration_hours": campaign_duration_hours, + "engine_used": engine_to_use, + "sanitizer_used": build_result.get("sanitizer_used") + } + results["crashes"] = all_crashes[:100] # Limit to first 100 crashes + + workflow.logger.info( + f"āœ“ Campaign completed: {project_name} - " + f"{total_execs} execs, {total_crashes} crashes" + ) + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + results["steps"].append({ + "step": "error", + "status": "failed", + "error": str(e) + }) + raise diff --git a/backend/toolbox/workflows/registry.py b/backend/toolbox/workflows/registry.py deleted file mode 100644 index ad58bc0..0000000 --- a/backend/toolbox/workflows/registry.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Manual Workflow Registry for Prefect Deployment - -This file contains the manual registry of all workflows that can be deployed. -Developers MUST add their workflows here after creating them. - -This approach is required because: -1. Prefect cannot deploy dynamically imported flows -2. Docker deployment needs static flow references -3. Explicit registration provides better control and visibility -""" - -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - -from typing import Dict, Any, Callable -import logging - -logger = logging.getLogger(__name__) - -# Import only essential workflows -# Import each workflow individually to handle failures gracefully -security_assessment_flow = None -secret_detection_flow = None - -# Try to import each workflow individually -try: - from .security_assessment.workflow import main_flow as security_assessment_flow -except ImportError as e: - logger.warning(f"Failed to import security_assessment workflow: {e}") - -try: - from .comprehensive.secret_detection_scan.workflow import main_flow as secret_detection_flow -except ImportError as e: - logger.warning(f"Failed to import secret_detection_scan workflow: {e}") - - -# Manual registry - developers add workflows here after creation -# Only include workflows that were successfully imported -WORKFLOW_REGISTRY: Dict[str, Dict[str, Any]] = {} - -# Add workflows that were successfully imported -if security_assessment_flow is not None: - WORKFLOW_REGISTRY["security_assessment"] = { - "flow": security_assessment_flow, - "module_path": "toolbox.workflows.security_assessment.workflow", - "function_name": "main_flow", - "description": "Comprehensive security assessment workflow that scans files, analyzes code for vulnerabilities, and generates SARIF reports", - "version": "1.0.0", - "author": "FuzzForge Team", - "tags": ["security", "scanner", "analyzer", "static-analysis", "sarif"] - } - -if secret_detection_flow is not None: - WORKFLOW_REGISTRY["secret_detection_scan"] = { - "flow": secret_detection_flow, - "module_path": "toolbox.workflows.comprehensive.secret_detection_scan.workflow", - "function_name": "main_flow", - "description": "Comprehensive secret detection using TruffleHog and Gitleaks for thorough credential scanning", - "version": "1.0.0", - "author": "FuzzForge Team", - "tags": ["secrets", "credentials", "detection", "trufflehog", "gitleaks", "comprehensive"] - } - -# -# To add a new workflow, follow this pattern: -# -# "my_new_workflow": { -# "flow": my_new_flow_function, # Import the flow function above -# "module_path": "toolbox.workflows.my_new_workflow.workflow", -# "function_name": "my_new_flow_function", -# "description": "Description of what this workflow does", -# "version": "1.0.0", -# "author": "Developer Name", -# "tags": ["tag1", "tag2"] -# } - - -def get_workflow_flow(workflow_name: str) -> Callable: - """ - Get the flow function for a workflow. - - Args: - workflow_name: Name of the workflow - - Returns: - Flow function - - Raises: - KeyError: If workflow not found in registry - """ - if workflow_name not in WORKFLOW_REGISTRY: - available = list(WORKFLOW_REGISTRY.keys()) - raise KeyError( - f"Workflow '{workflow_name}' not found in registry. " - f"Available workflows: {available}. " - f"Please add the workflow to toolbox/workflows/registry.py" - ) - - return WORKFLOW_REGISTRY[workflow_name]["flow"] - - -def get_workflow_info(workflow_name: str) -> Dict[str, Any]: - """ - Get registry information for a workflow. - - Args: - workflow_name: Name of the workflow - - Returns: - Registry information dictionary - - Raises: - KeyError: If workflow not found in registry - """ - if workflow_name not in WORKFLOW_REGISTRY: - available = list(WORKFLOW_REGISTRY.keys()) - raise KeyError( - f"Workflow '{workflow_name}' not found in registry. " - f"Available workflows: {available}" - ) - - return WORKFLOW_REGISTRY[workflow_name] - - -def list_registered_workflows() -> Dict[str, Dict[str, Any]]: - """ - Get all registered workflows. - - Returns: - Dictionary of all workflow registry entries - """ - return WORKFLOW_REGISTRY.copy() - - -def validate_registry() -> bool: - """ - Validate the workflow registry for consistency. - - Returns: - True if valid, raises exceptions if not - - Raises: - ValueError: If registry is invalid - """ - if not WORKFLOW_REGISTRY: - raise ValueError("Workflow registry is empty") - - required_fields = ["flow", "module_path", "function_name", "description"] - - for name, entry in WORKFLOW_REGISTRY.items(): - # Check required fields - missing_fields = [field for field in required_fields if field not in entry] - if missing_fields: - raise ValueError( - f"Workflow '{name}' missing required fields: {missing_fields}" - ) - - # Check if flow is callable - if not callable(entry["flow"]): - raise ValueError(f"Workflow '{name}' flow is not callable") - - # Check if flow has the required Prefect attributes - if not hasattr(entry["flow"], "deploy"): - raise ValueError( - f"Workflow '{name}' flow is not a Prefect flow (missing deploy method)" - ) - - logger.info(f"Registry validation passed. {len(WORKFLOW_REGISTRY)} workflows registered.") - return True - - -# Validate registry on import -try: - validate_registry() - logger.info(f"Workflow registry loaded successfully with {len(WORKFLOW_REGISTRY)} workflows") -except Exception as e: - logger.error(f"Workflow registry validation failed: {e}") - raise \ No newline at end of file diff --git a/backend/toolbox/workflows/security_assessment/Dockerfile b/backend/toolbox/workflows/security_assessment/Dockerfile deleted file mode 100644 index 2b46c2c..0000000 --- a/backend/toolbox/workflows/security_assessment/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM prefecthq/prefect:3-python3.11 - -WORKDIR /app - -# Create toolbox directory structure to match expected import paths -RUN mkdir -p /app/toolbox/workflows /app/toolbox/modules - -# Copy base module infrastructure -COPY modules/__init__.py /app/toolbox/modules/ -COPY modules/base.py /app/toolbox/modules/ - -# Copy only required modules (manual selection) -COPY modules/scanner /app/toolbox/modules/scanner -COPY modules/analyzer /app/toolbox/modules/analyzer -COPY modules/reporter /app/toolbox/modules/reporter - -# Copy this workflow -COPY workflows/security_assessment /app/toolbox/workflows/security_assessment - -# Install workflow-specific requirements if they exist -RUN if [ -f /app/toolbox/workflows/security_assessment/requirements.txt ]; then pip install --no-cache-dir -r /app/toolbox/workflows/security_assessment/requirements.txt; fi - -# Install common requirements -RUN pip install --no-cache-dir pyyaml - -# Set Python path -ENV PYTHONPATH=/app:$PYTHONPATH - -# Create workspace directory -RUN mkdir -p /workspace diff --git a/backend/toolbox/workflows/security_assessment/activities.py b/backend/toolbox/workflows/security_assessment/activities.py new file mode 100644 index 0000000..ca9182f --- /dev/null +++ b/backend/toolbox/workflows/security_assessment/activities.py @@ -0,0 +1,150 @@ +""" +Security Assessment Workflow Activities + +Activities specific to the security assessment workflow: +- scan_files_activity: Scan files in the workspace +- analyze_security_activity: Analyze security vulnerabilities +- generate_sarif_report_activity: Generate SARIF report from findings +""" + +import logging +import sys +from pathlib import Path + +from temporalio import activity + +# Configure logging +logger = logging.getLogger(__name__) + +# Add toolbox to path for module imports +sys.path.insert(0, '/app/toolbox') + + +@activity.defn(name="scan_files") +async def scan_files_activity(workspace_path: str, config: dict) -> dict: + """ + Scan files in the workspace. + + Args: + workspace_path: Path to the workspace directory + config: Scanner configuration + + Returns: + Scanner results dictionary + """ + logger.info(f"Activity: scan_files (workspace={workspace_path})") + + try: + from modules.scanner import FileScanner + + workspace = Path(workspace_path) + if not workspace.exists(): + raise FileNotFoundError(f"Workspace not found: {workspace_path}") + + scanner = FileScanner() + result = await scanner.execute(config, workspace) + + logger.info( + f"āœ“ File scanning completed: " + f"{result.summary.get('total_files', 0)} files scanned" + ) + return result.dict() + + except Exception as e: + logger.error(f"File scanning failed: {e}", exc_info=True) + raise + + +@activity.defn(name="analyze_security") +async def analyze_security_activity(workspace_path: str, config: dict) -> dict: + """ + Analyze security vulnerabilities in the workspace. + + Args: + workspace_path: Path to the workspace directory + config: Analyzer configuration + + Returns: + Analysis results dictionary + """ + logger.info(f"Activity: analyze_security (workspace={workspace_path})") + + try: + from modules.analyzer import SecurityAnalyzer + + workspace = Path(workspace_path) + if not workspace.exists(): + raise FileNotFoundError(f"Workspace not found: {workspace_path}") + + analyzer = SecurityAnalyzer() + result = await analyzer.execute(config, workspace) + + logger.info( + f"āœ“ Security analysis completed: " + f"{result.summary.get('total_findings', 0)} findings" + ) + return result.dict() + + except Exception as e: + logger.error(f"Security analysis failed: {e}", exc_info=True) + raise + + +@activity.defn(name="generate_sarif_report") +async def generate_sarif_report_activity( + scan_results: dict, + analysis_results: dict, + config: dict, + workspace_path: str +) -> dict: + """ + Generate SARIF report from scan and analysis results. + + Args: + scan_results: Results from file scanner + analysis_results: Results from security analyzer + config: Reporter configuration + workspace_path: Path to the workspace + + Returns: + SARIF report dictionary + """ + logger.info("Activity: generate_sarif_report") + + try: + from modules.reporter import SARIFReporter + + workspace = Path(workspace_path) + + # Combine findings from all modules + all_findings = [] + + # Add scanner findings (only sensitive files, not all files) + scanner_findings = scan_results.get("findings", []) + sensitive_findings = [f for f in scanner_findings if f.get("severity") != "info"] + all_findings.extend(sensitive_findings) + + # Add analyzer findings + analyzer_findings = analysis_results.get("findings", []) + all_findings.extend(analyzer_findings) + + # Prepare reporter config + reporter_config = { + **config, + "findings": all_findings, + "tool_name": "FuzzForge Security Assessment", + "tool_version": "1.0.0" + } + + reporter = SARIFReporter() + result = await reporter.execute(reporter_config, workspace) + + # Extract SARIF from result + sarif = result.dict().get("sarif", {}) + + logger.info(f"āœ“ SARIF report generated with {len(all_findings)} findings") + return sarif + + except Exception as e: + logger.error(f"SARIF report generation failed: {e}", exc_info=True) + raise diff --git a/backend/toolbox/workflows/security_assessment/metadata.yaml b/backend/toolbox/workflows/security_assessment/metadata.yaml index e3ffbe8..572e50c 100644 --- a/backend/toolbox/workflows/security_assessment/metadata.yaml +++ b/backend/toolbox/workflows/security_assessment/metadata.yaml @@ -1,8 +1,8 @@ name: security_assessment version: "2.0.0" +vertical: python description: "Comprehensive security assessment workflow that scans files, analyzes code for vulnerabilities, and generates SARIF reports" author: "FuzzForge Team" -category: "comprehensive" tags: - "security" - "scanner" @@ -11,28 +11,14 @@ tags: - "sarif" - "comprehensive" -supported_volume_modes: - - "ro" - - "rw" - -default_volume_mode: "ro" -default_target_path: "/workspace" - -requirements: - tools: - - "file_scanner" - - "security_analyzer" - - "sarif_reporter" - resources: - memory: "512Mi" - cpu: "500m" - timeout: 1800 - -has_docker: true +# Workspace isolation mode (system-level configuration) +# - "isolated" (default): Each workflow run gets its own isolated workspace (safe for concurrent fuzzing) +# - "shared": All runs share the same workspace (for read-only analysis workflows) +# - "copy-on-write": Download once, copy for each run (balances performance and isolation) +# Using "shared" mode for read-only security analysis (no file modifications) +workspace_isolation: "shared" default_parameters: - target_path: "/workspace" - volume_mode: "ro" scanner_config: {} analyzer_config: {} reporter_config: {} @@ -40,15 +26,6 @@ default_parameters: parameters: type: object properties: - target_path: - type: string - default: "/workspace" - description: "Path to analyze" - volume_mode: - type: string - enum: ["ro", "rw"] - default: "ro" - description: "Volume mount mode" scanner_config: type: object description: "File scanner configuration" diff --git a/backend/toolbox/workflows/security_assessment/requirements.txt b/backend/toolbox/workflows/security_assessment/requirements.txt deleted file mode 100644 index f481334..0000000 --- a/backend/toolbox/workflows/security_assessment/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Requirements for security assessment workflow -pydantic>=2.0.0 -pyyaml>=6.0 -aiofiles>=23.0.0 \ No newline at end of file diff --git a/backend/toolbox/workflows/security_assessment/workflow.py b/backend/toolbox/workflows/security_assessment/workflow.py index 584bf65..d7ff21c 100644 --- a/backend/toolbox/workflows/security_assessment/workflow.py +++ b/backend/toolbox/workflows/security_assessment/workflow.py @@ -1,5 +1,7 @@ """ -Security Assessment Workflow - Comprehensive security analysis using multiple modules +Security Assessment Workflow - Temporal Version + +Comprehensive security analysis using multiple modules. """ # Copyright (c) 2025 FuzzingLabs @@ -13,240 +15,219 @@ Security Assessment Workflow - Comprehensive security analysis using multiple mo # # Additional attribution and requirements are provided in the NOTICE file. -import sys -import logging -from pathlib import Path +from datetime import timedelta from typing import Dict, Any, Optional -from prefect import flow, task -import json -# Add modules to path -sys.path.insert(0, '/app') +from temporalio import workflow +from temporalio.common import RetryPolicy -# Import modules -from toolbox.modules.scanner import FileScanner -from toolbox.modules.analyzer import SecurityAnalyzer -from toolbox.modules.reporter import SARIFReporter +# Import activity interfaces (will be executed by worker) +with workflow.unsafe.imports_passed_through(): + import logging -# Configure logging -logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -@task(name="file_scanning") -async def scan_files_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: +@workflow.defn +class SecurityAssessmentWorkflow: """ - Task to scan files in the workspace. - - Args: - workspace: Path to the workspace - config: Scanner configuration - - Returns: - Scanner results - """ - logger.info(f"Starting file scanning in {workspace}") - scanner = FileScanner() - - result = await scanner.execute(config, workspace) - - logger.info(f"File scanning completed: {result.summary.get('total_files', 0)} files found") - return result.dict() - - -@task(name="security_analysis") -async def analyze_security_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: - """ - Task to analyze security vulnerabilities. - - Args: - workspace: Path to the workspace - config: Analyzer configuration - - Returns: - Analysis results - """ - logger.info("Starting security analysis") - analyzer = SecurityAnalyzer() - - result = await analyzer.execute(config, workspace) - - logger.info( - f"Security analysis completed: {result.summary.get('total_findings', 0)} findings" - ) - return result.dict() - - -@task(name="report_generation") -async def generate_report_task( - scan_results: Dict[str, Any], - analysis_results: Dict[str, Any], - config: Dict[str, Any], - workspace: Path -) -> Dict[str, Any]: - """ - Task to generate SARIF report from all findings. - - Args: - scan_results: Results from scanner - analysis_results: Results from analyzer - config: Reporter configuration - workspace: Path to the workspace - - Returns: - SARIF report - """ - logger.info("Generating SARIF report") - reporter = SARIFReporter() - - # Combine findings from all modules - all_findings = [] - - # Add scanner findings (only sensitive files, not all files) - scanner_findings = scan_results.get("findings", []) - sensitive_findings = [f for f in scanner_findings if f.get("severity") != "info"] - all_findings.extend(sensitive_findings) - - # Add analyzer findings - analyzer_findings = analysis_results.get("findings", []) - all_findings.extend(analyzer_findings) - - # Prepare reporter config - reporter_config = { - **config, - "findings": all_findings, - "tool_name": "FuzzForge Security Assessment", - "tool_version": "1.0.0" - } - - result = await reporter.execute(reporter_config, workspace) - - # Extract SARIF from result - sarif = result.dict().get("sarif", {}) - - logger.info(f"Report generated with {len(all_findings)} total findings") - return sarif - - -@flow(name="security_assessment", log_prints=True) -async def main_flow( - target_path: str = "/workspace", - volume_mode: str = "ro", - scanner_config: Optional[Dict[str, Any]] = None, - analyzer_config: Optional[Dict[str, Any]] = None, - reporter_config: Optional[Dict[str, Any]] = None -) -> Dict[str, Any]: - """ - Main security assessment workflow. + Comprehensive security assessment workflow. This workflow: - 1. Scans files in the workspace - 2. Analyzes code for security vulnerabilities - 3. Generates a SARIF report with all findings - - Args: - target_path: Path to the mounted workspace (default: /workspace) - volume_mode: Volume mount mode (ro/rw) - scanner_config: Configuration for file scanner - analyzer_config: Configuration for security analyzer - reporter_config: Configuration for SARIF reporter - - Returns: - SARIF-formatted findings report + 1. Downloads target from MinIO + 2. Scans files in the workspace + 3. Analyzes code for security vulnerabilities + 4. Generates a SARIF report with all findings + 5. Uploads results to MinIO + 6. Cleans up cache """ - logger.info(f"Starting security assessment workflow") - logger.info(f"Workspace: {target_path}, Mode: {volume_mode}") - # Set workspace path - workspace = Path(target_path) + @workflow.run + async def run( + self, + target_id: str, + scanner_config: Optional[Dict[str, Any]] = None, + analyzer_config: Optional[Dict[str, Any]] = None, + reporter_config: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Main workflow execution. - if not workspace.exists(): - logger.error(f"Workspace does not exist: {workspace}") - return { - "error": f"Workspace not found: {workspace}", - "sarif": None - } + Args: + target_id: UUID of the uploaded target in MinIO + scanner_config: Configuration for file scanner + analyzer_config: Configuration for security analyzer + reporter_config: Configuration for SARIF reporter - # Default configurations - if not scanner_config: - scanner_config = { - "patterns": ["*"], - "check_sensitive": True, - "calculate_hashes": False, - "max_file_size": 10485760 # 10MB - } + Returns: + Dictionary containing SARIF report and summary + """ + workflow_id = workflow.info().workflow_id - if not analyzer_config: - analyzer_config = { - "file_extensions": [".py", ".js", ".java", ".php", ".rb", ".go"], - "check_secrets": True, - "check_sql": True, - "check_dangerous_functions": True - } - - if not reporter_config: - reporter_config = { - "include_code_flows": False - } - - try: - # Execute workflow tasks - logger.info("Phase 1: File scanning") - scan_results = await scan_files_task(workspace, scanner_config) - - logger.info("Phase 2: Security analysis") - analysis_results = await analyze_security_task(workspace, analyzer_config) - - logger.info("Phase 3: Report generation") - sarif_report = await generate_report_task( - scan_results, - analysis_results, - reporter_config, - workspace + workflow.logger.info( + f"Starting SecurityAssessmentWorkflow " + f"(workflow_id={workflow_id}, target_id={target_id})" ) - # Log summary - if sarif_report and "runs" in sarif_report: - results_count = len(sarif_report["runs"][0].get("results", [])) - logger.info(f"Workflow completed successfully with {results_count} findings") - else: - logger.info("Workflow completed successfully") + # Default configurations + if not scanner_config: + scanner_config = { + "patterns": ["*"], + "check_sensitive": True, + "calculate_hashes": False, + "max_file_size": 10485760 # 10MB + } - return sarif_report + if not analyzer_config: + analyzer_config = { + "file_extensions": [".py", ".js", ".java", ".php", ".rb", ".go"], + "check_secrets": True, + "check_sql": True, + "check_dangerous_functions": True + } - except Exception as e: - logger.error(f"Workflow failed: {e}") - # Return error in SARIF format - return { - "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", - "version": "2.1.0", - "runs": [ - { - "tool": { - "driver": { - "name": "FuzzForge Security Assessment", - "version": "1.0.0" - } - }, - "results": [], - "invocations": [ - { - "executionSuccessful": False, - "exitCode": 1, - "exitCodeDescription": str(e) - } - ] - } - ] + if not reporter_config: + reporter_config = { + "include_code_flows": False + } + + results = { + "workflow_id": workflow_id, + "target_id": target_id, + "status": "running", + "steps": [] } + try: + # Get run ID for workspace isolation (using shared mode for read-only analysis) + run_id = workflow.info().run_id -if __name__ == "__main__": - # For local testing - import asyncio + # Step 1: Download target from MinIO + workflow.logger.info("Step 1: Downloading target from MinIO") + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "shared"], # target_id, run_id, workspace_isolation + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + results["steps"].append({ + "step": "download_target", + "status": "success", + "target_path": target_path + }) + workflow.logger.info(f"āœ“ Target downloaded to: {target_path}") - asyncio.run(main_flow( - target_path="/tmp/test", - scanner_config={"patterns": ["*.py"]}, - analyzer_config={"check_secrets": True} - )) \ No newline at end of file + # Step 2: File scanning + workflow.logger.info("Step 2: Scanning files") + scan_results = await workflow.execute_activity( + "scan_files", + args=[target_path, scanner_config], + start_to_close_timeout=timedelta(minutes=10), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=2 + ) + ) + results["steps"].append({ + "step": "file_scanning", + "status": "success", + "files_scanned": scan_results.get("summary", {}).get("total_files", 0) + }) + workflow.logger.info( + f"āœ“ File scanning completed: " + f"{scan_results.get('summary', {}).get('total_files', 0)} files" + ) + + # Step 3: Security analysis + workflow.logger.info("Step 3: Analyzing security vulnerabilities") + analysis_results = await workflow.execute_activity( + "analyze_security", + args=[target_path, analyzer_config], + start_to_close_timeout=timedelta(minutes=15), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=2 + ) + ) + results["steps"].append({ + "step": "security_analysis", + "status": "success", + "findings": analysis_results.get("summary", {}).get("total_findings", 0) + }) + workflow.logger.info( + f"āœ“ Security analysis completed: " + f"{analysis_results.get('summary', {}).get('total_findings', 0)} findings" + ) + + # Step 4: Generate SARIF report + workflow.logger.info("Step 4: Generating SARIF report") + sarif_report = await workflow.execute_activity( + "generate_sarif_report", + args=[scan_results, analysis_results, reporter_config, target_path], + start_to_close_timeout=timedelta(minutes=5) + ) + results["steps"].append({ + "step": "report_generation", + "status": "success" + }) + + # Count total findings in SARIF + total_findings = 0 + if sarif_report and "runs" in sarif_report: + total_findings = len(sarif_report["runs"][0].get("results", [])) + + workflow.logger.info(f"āœ“ SARIF report generated with {total_findings} findings") + + # Step 5: Upload results to MinIO + workflow.logger.info("Step 5: Uploading results") + try: + results_url = await workflow.execute_activity( + "upload_results", + args=[workflow_id, sarif_report, "sarif"], + start_to_close_timeout=timedelta(minutes=2) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + except Exception as e: + workflow.logger.warning(f"Failed to upload results: {e}") + results["results_url"] = None + + # Step 6: Cleanup cache + workflow.logger.info("Step 6: Cleaning up cache") + try: + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "shared"], # target_path, workspace_isolation + start_to_close_timeout=timedelta(minutes=1) + ) + workflow.logger.info("āœ“ Cache cleaned up (skipped for shared mode)") + except Exception as e: + workflow.logger.warning(f"Cache cleanup failed: {e}") + + # Mark workflow as successful + results["status"] = "success" + results["sarif"] = sarif_report + results["summary"] = { + "total_findings": total_findings, + "files_scanned": scan_results.get("summary", {}).get("total_files", 0) + } + workflow.logger.info(f"āœ“ Workflow completed successfully: {workflow_id}") + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + results["steps"].append({ + "step": "error", + "status": "failed", + "error": str(e) + }) + raise diff --git a/backend/toolbox/workflows/trufflehog_detection/__init__.py b/backend/toolbox/workflows/trufflehog_detection/__init__.py new file mode 100644 index 0000000..d580fb8 --- /dev/null +++ b/backend/toolbox/workflows/trufflehog_detection/__init__.py @@ -0,0 +1,13 @@ +""" +TruffleHog Detection Workflow +""" + +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. + +from .workflow import TrufflehogDetectionWorkflow +from .activities import scan_with_trufflehog, trufflehog_generate_sarif + +__all__ = ["TrufflehogDetectionWorkflow", "scan_with_trufflehog", "trufflehog_generate_sarif"] diff --git a/backend/toolbox/workflows/trufflehog_detection/activities.py b/backend/toolbox/workflows/trufflehog_detection/activities.py new file mode 100644 index 0000000..31bb92c --- /dev/null +++ b/backend/toolbox/workflows/trufflehog_detection/activities.py @@ -0,0 +1,110 @@ +"""TruffleHog Detection Workflow Activities""" + +from pathlib import Path +from typing import Dict, Any +from temporalio import activity + +try: + from toolbox.modules.secret_detection.trufflehog import TruffleHogModule +except ImportError: + from modules.secret_detection.trufflehog import TruffleHogModule + +@activity.defn(name="scan_with_trufflehog") +async def scan_with_trufflehog(target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: + """Scan code using TruffleHog.""" + activity.logger.info(f"Starting TruffleHog scan: {target_path}") + workspace = Path(target_path) + + trufflehog = TruffleHogModule() + trufflehog.validate_config(config) + result = await trufflehog.execute(config, workspace) + + if result.status == "failed": + raise RuntimeError(f"TruffleHog scan failed: {result.error}") + + findings_dicts = [finding.model_dump() for finding in result.findings] + return {"findings": findings_dicts, "summary": result.summary} + + +@activity.defn(name="trufflehog_generate_sarif") +async def trufflehog_generate_sarif(findings: list, metadata: Dict[str, Any]) -> Dict[str, Any]: + """ + Generate SARIF report from TruffleHog findings. + + Args: + findings: List of finding dictionaries + metadata: Metadata including tool_name, tool_version + + Returns: + SARIF report dictionary + """ + activity.logger.info(f"Generating SARIF report from {len(findings)} findings") + + # Basic SARIF 2.1.0 structure + sarif_report = { + "version": "2.1.0", + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": metadata.get("tool_name", "trufflehog"), + "version": metadata.get("tool_version", "3.63.2"), + "informationUri": "https://github.com/trufflesecurity/trufflehog" + } + }, + "results": [] + } + ] + } + + # Convert findings to SARIF results + for finding in findings: + sarif_result = { + "ruleId": finding.get("metadata", {}).get("detector", "unknown"), + "level": _severity_to_sarif_level(finding.get("severity", "warning")), + "message": { + "text": finding.get("title", "Secret detected") + }, + "locations": [] + } + + # Add description if present + if finding.get("description"): + sarif_result["message"]["markdown"] = finding["description"] + + # Add location if file path is present + if finding.get("file_path"): + location = { + "physicalLocation": { + "artifactLocation": { + "uri": finding["file_path"] + } + } + } + + # Add region if line number is present + if finding.get("line_start"): + location["physicalLocation"]["region"] = { + "startLine": finding["line_start"] + } + + sarif_result["locations"].append(location) + + sarif_report["runs"][0]["results"].append(sarif_result) + + activity.logger.info(f"Generated SARIF report with {len(sarif_report['runs'][0]['results'])} results") + + return sarif_report + + +def _severity_to_sarif_level(severity: str) -> str: + """Convert severity to SARIF level""" + severity_map = { + "critical": "error", + "high": "error", + "medium": "warning", + "low": "note", + "info": "note" + } + return severity_map.get(severity.lower(), "warning") diff --git a/backend/toolbox/workflows/trufflehog_detection/metadata.yaml b/backend/toolbox/workflows/trufflehog_detection/metadata.yaml new file mode 100644 index 0000000..1a147f0 --- /dev/null +++ b/backend/toolbox/workflows/trufflehog_detection/metadata.yaml @@ -0,0 +1,34 @@ +name: trufflehog_detection +version: "1.0.0" +vertical: secrets +description: "Detect secrets with verification using TruffleHog" +author: "FuzzForge Team" +tags: + - "secrets" + - "trufflehog" + - "verification" + +workspace_isolation: "shared" + +parameters: + type: object + properties: + verify: + type: boolean + default: true + description: "Verify discovered secrets" + + max_depth: + type: integer + default: 10 + description: "Maximum directory depth to scan" + +default_parameters: + verify: true + max_depth: 10 + +required_modules: + - "trufflehog" + +supported_volume_modes: + - "ro" diff --git a/backend/toolbox/workflows/trufflehog_detection/workflow.py b/backend/toolbox/workflows/trufflehog_detection/workflow.py new file mode 100644 index 0000000..62336f3 --- /dev/null +++ b/backend/toolbox/workflows/trufflehog_detection/workflow.py @@ -0,0 +1,104 @@ +"""TruffleHog Detection Workflow""" + +from datetime import timedelta +from typing import Dict, Any +from temporalio import workflow +from temporalio.common import RetryPolicy + +@workflow.defn +class TrufflehogDetectionWorkflow: + """Scan code for secrets using TruffleHog.""" + + @workflow.run + async def run(self, target_id: str, verify: bool = False, concurrency: int = 10) -> Dict[str, Any]: + workflow_id = workflow.info().workflow_id + run_id = workflow.info().run_id + + workflow.logger.info( + f"Starting TrufflehogDetectionWorkflow " + f"(workflow_id={workflow_id}, target_id={target_id}, verify={verify})" + ) + + results = {"workflow_id": workflow_id, "status": "running", "findings": []} + + try: + # Step 1: Download target + workflow.logger.info("Step 1: Downloading target from MinIO") + target_path = await workflow.execute_activity( + "get_target", args=[target_id, run_id, "shared"], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=1), + maximum_interval=timedelta(seconds=30), + maximum_attempts=3 + ) + ) + workflow.logger.info(f"āœ“ Target downloaded to: {target_path}") + + # Step 2: Scan with TruffleHog + workflow.logger.info("Step 2: Scanning with TruffleHog") + scan_results = await workflow.execute_activity( + "scan_with_trufflehog", + args=[target_path, {"verify": verify, "concurrency": concurrency}], + start_to_close_timeout=timedelta(minutes=15), + retry_policy=RetryPolicy( + initial_interval=timedelta(seconds=2), + maximum_interval=timedelta(seconds=60), + maximum_attempts=2 + ) + ) + workflow.logger.info( + f"āœ“ TruffleHog scan completed: " + f"{scan_results.get('summary', {}).get('total_secrets', 0)} secrets found" + ) + + # Step 3: Generate SARIF report + workflow.logger.info("Step 3: Generating SARIF report") + sarif_report = await workflow.execute_activity( + "trufflehog_generate_sarif", + args=[scan_results.get("findings", []), {"tool_name": "trufflehog", "tool_version": "3.63.2"}], + start_to_close_timeout=timedelta(minutes=2) + ) + + # Step 4: Upload results to MinIO + workflow.logger.info("Step 4: Uploading results") + try: + results_url = await workflow.execute_activity( + "upload_results", + args=[workflow_id, scan_results, "json"], + start_to_close_timeout=timedelta(minutes=2) + ) + results["results_url"] = results_url + workflow.logger.info(f"āœ“ Results uploaded to: {results_url}") + except Exception as e: + workflow.logger.warning(f"Failed to upload results: {e}") + results["results_url"] = None + + # Step 5: Cleanup + workflow.logger.info("Step 5: Cleaning up cache") + try: + await workflow.execute_activity( + "cleanup_cache", args=[target_path, "shared"], + start_to_close_timeout=timedelta(minutes=1) + ) + workflow.logger.info("āœ“ Cache cleaned up") + except Exception as e: + workflow.logger.warning(f"Cache cleanup failed: {e}") + + # Mark workflow as successful + results["status"] = "success" + results["findings"] = scan_results.get("findings", []) + results["summary"] = scan_results.get("summary", {}) + results["sarif"] = sarif_report or {} + workflow.logger.info( + f"āœ“ Workflow completed successfully: {workflow_id} " + f"({results['summary'].get('total_secrets', 0)} secrets found)" + ) + + return results + + except Exception as e: + workflow.logger.error(f"Workflow failed: {e}") + results["status"] = "error" + results["error"] = str(e) + raise diff --git a/backend/uv.lock b/backend/uv.lock index 6753e50..82803c8 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -105,32 +105,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] -[[package]] -name = "aiosqlite" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454, upload-time = "2025-02-03T07:30:16.235Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, -] - -[[package]] -name = "alembic" -version = "1.16.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mako" }, - { name = "sqlalchemy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9a/ca/4dc52902cf3491892d464f5265a81e9dff094692c8a049a3ed6a05fe7ee8/alembic-1.16.5.tar.gz", hash = "sha256:a88bb7f6e513bd4301ecf4c7f2206fe93f9913f9b48dac3b78babde2d6fe765e", size = 1969868, upload-time = "2025-08-27T18:02:05.668Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/4a/4c61d4c84cfd9befb6fa08a702535b27b21fff08c946bc2f6139decbf7f7/alembic-1.16.5-py3-none-any.whl", hash = "sha256:e845dfe090c5ffa7b92593ae6687c5cb1a101e91fa53868497dbd79847f9dbe3", size = 247355, upload-time = "2025-08-27T18:02:07.37Z" }, -] - [[package]] name = "annotated-types" version = "0.7.0" @@ -154,67 +128,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, ] -[[package]] -name = "apprise" -version = "1.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "click" }, - { name = "markdown" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "requests-oauthlib" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/51/f9/bda66afaf393f6914f4d6c035964936cadd98ee1fef44e4e77cba3b5828c/apprise-1.9.4.tar.gz", hash = "sha256:483122aee19a89a7b075ecd48ef11ae37d79744f7aeb450bcf985a9a6c28c988", size = 1855012, upload-time = "2025-08-02T18:13:28.467Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/fa/7875ad63088b2d7dea538ffe60fba85786c228c7349d258891c54d0416a0/apprise-1.9.4-py3-none-any.whl", hash = "sha256:17dca8ad0a5b2063f6bae707979a51ca2cb374fcc66b0dd5c05a9d286dd40069", size = 1402630, upload-time = "2025-08-02T18:13:26.263Z" }, -] - -[[package]] -name = "asgi-lifespan" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sniffio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/da/e7908b54e0f8043725a990bf625f2041ecf6bfe8eb7b19407f1c00b630f7/asgi-lifespan-2.1.0.tar.gz", hash = "sha256:5e2effaf0bfe39829cf2d64e7ecc47c7d86d676a6599f7afba378c31f5e3a308", size = 15627, upload-time = "2023-03-28T17:35:49.126Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/f5/c36551e93acba41a59939ae6a0fb77ddb3f2e8e8caa716410c65f7341f72/asgi_lifespan-2.1.0-py3-none-any.whl", hash = "sha256:ed840706680e28428c01e14afb3875d7d76d3206f3d5b2f2294e059b5c23804f", size = 10895, upload-time = "2023-03-28T17:35:47.772Z" }, -] - -[[package]] -name = "asyncpg" -version = "0.30.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/4c/7c991e080e106d854809030d8584e15b2e996e26f16aee6d757e387bc17d/asyncpg-0.30.0.tar.gz", hash = "sha256:c551e9928ab6707602f44811817f82ba3c446e018bfe1d3abecc8ba5f3eac851", size = 957746, upload-time = "2024-10-20T00:30:41.127Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/0e/f5d708add0d0b97446c402db7e8dd4c4183c13edaabe8a8500b411e7b495/asyncpg-0.30.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5e0511ad3dec5f6b4f7a9e063591d407eee66b88c14e2ea636f187da1dcfff6a", size = 674506, upload-time = "2024-10-20T00:29:27.988Z" }, - { url = "https://files.pythonhosted.org/packages/6a/a0/67ec9a75cb24a1d99f97b8437c8d56da40e6f6bd23b04e2f4ea5d5ad82ac/asyncpg-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:915aeb9f79316b43c3207363af12d0e6fd10776641a7de8a01212afd95bdf0ed", size = 645922, upload-time = "2024-10-20T00:29:29.391Z" }, - { url = "https://files.pythonhosted.org/packages/5c/d9/a7584f24174bd86ff1053b14bb841f9e714380c672f61c906eb01d8ec433/asyncpg-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c198a00cce9506fcd0bf219a799f38ac7a237745e1d27f0e1f66d3707c84a5a", size = 3079565, upload-time = "2024-10-20T00:29:30.832Z" }, - { url = "https://files.pythonhosted.org/packages/a0/d7/a4c0f9660e333114bdb04d1a9ac70db690dd4ae003f34f691139a5cbdae3/asyncpg-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3326e6d7381799e9735ca2ec9fd7be4d5fef5dcbc3cb555d8a463d8460607956", size = 3109962, upload-time = "2024-10-20T00:29:33.114Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/199fd16b5a981b1575923cbb5d9cf916fdc936b377e0423099f209e7e73d/asyncpg-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:51da377487e249e35bd0859661f6ee2b81db11ad1f4fc036194bc9cb2ead5056", size = 3064791, upload-time = "2024-10-20T00:29:34.677Z" }, - { url = "https://files.pythonhosted.org/packages/77/52/0004809b3427534a0c9139c08c87b515f1c77a8376a50ae29f001e53962f/asyncpg-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc6d84136f9c4d24d358f3b02be4b6ba358abd09f80737d1ac7c444f36108454", size = 3188696, upload-time = "2024-10-20T00:29:36.389Z" }, - { url = "https://files.pythonhosted.org/packages/52/cb/fbad941cd466117be58b774a3f1cc9ecc659af625f028b163b1e646a55fe/asyncpg-0.30.0-cp311-cp311-win32.whl", hash = "sha256:574156480df14f64c2d76450a3f3aaaf26105869cad3865041156b38459e935d", size = 567358, upload-time = "2024-10-20T00:29:37.915Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0a/0a32307cf166d50e1ad120d9b81a33a948a1a5463ebfa5a96cc5606c0863/asyncpg-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:3356637f0bd830407b5597317b3cb3571387ae52ddc3bca6233682be88bbbc1f", size = 629375, upload-time = "2024-10-20T00:29:39.987Z" }, - { url = "https://files.pythonhosted.org/packages/4b/64/9d3e887bb7b01535fdbc45fbd5f0a8447539833b97ee69ecdbb7a79d0cb4/asyncpg-0.30.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c902a60b52e506d38d7e80e0dd5399f657220f24635fee368117b8b5fce1142e", size = 673162, upload-time = "2024-10-20T00:29:41.88Z" }, - { url = "https://files.pythonhosted.org/packages/6e/eb/8b236663f06984f212a087b3e849731f917ab80f84450e943900e8ca4052/asyncpg-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aca1548e43bbb9f0f627a04666fedaca23db0a31a84136ad1f868cb15deb6e3a", size = 637025, upload-time = "2024-10-20T00:29:43.352Z" }, - { url = "https://files.pythonhosted.org/packages/cc/57/2dc240bb263d58786cfaa60920779af6e8d32da63ab9ffc09f8312bd7a14/asyncpg-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c2a2ef565400234a633da0eafdce27e843836256d40705d83ab7ec42074efb3", size = 3496243, upload-time = "2024-10-20T00:29:44.922Z" }, - { url = "https://files.pythonhosted.org/packages/f4/40/0ae9d061d278b10713ea9021ef6b703ec44698fe32178715a501ac696c6b/asyncpg-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1292b84ee06ac8a2ad8e51c7475aa309245874b61333d97411aab835c4a2f737", size = 3575059, upload-time = "2024-10-20T00:29:46.891Z" }, - { url = "https://files.pythonhosted.org/packages/c3/75/d6b895a35a2c6506952247640178e5f768eeb28b2e20299b6a6f1d743ba0/asyncpg-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f5712350388d0cd0615caec629ad53c81e506b1abaaf8d14c93f54b35e3595a", size = 3473596, upload-time = "2024-10-20T00:29:49.201Z" }, - { url = "https://files.pythonhosted.org/packages/c8/e7/3693392d3e168ab0aebb2d361431375bd22ffc7b4a586a0fc060d519fae7/asyncpg-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db9891e2d76e6f425746c5d2da01921e9a16b5a71a1c905b13f30e12a257c4af", size = 3641632, upload-time = "2024-10-20T00:29:50.768Z" }, - { url = "https://files.pythonhosted.org/packages/32/ea/15670cea95745bba3f0352341db55f506a820b21c619ee66b7d12ea7867d/asyncpg-0.30.0-cp312-cp312-win32.whl", hash = "sha256:68d71a1be3d83d0570049cd1654a9bdfe506e794ecc98ad0873304a9f35e411e", size = 560186, upload-time = "2024-10-20T00:29:52.394Z" }, - { url = "https://files.pythonhosted.org/packages/7e/6b/fe1fad5cee79ca5f5c27aed7bd95baee529c1bf8a387435c8ba4fe53d5c1/asyncpg-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a0292c6af5c500523949155ec17b7fe01a00ace33b68a476d6b5059f9630305", size = 621064, upload-time = "2024-10-20T00:29:53.757Z" }, - { url = "https://files.pythonhosted.org/packages/3a/22/e20602e1218dc07692acf70d5b902be820168d6282e69ef0d3cb920dc36f/asyncpg-0.30.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05b185ebb8083c8568ea8a40e896d5f7af4b8554b64d7719c0eaa1eb5a5c3a70", size = 670373, upload-time = "2024-10-20T00:29:55.165Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b3/0cf269a9d647852a95c06eb00b815d0b95a4eb4b55aa2d6ba680971733b9/asyncpg-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c47806b1a8cbb0a0db896f4cd34d89942effe353a5035c62734ab13b9f938da3", size = 634745, upload-time = "2024-10-20T00:29:57.14Z" }, - { url = "https://files.pythonhosted.org/packages/8e/6d/a4f31bf358ce8491d2a31bfe0d7bcf25269e80481e49de4d8616c4295a34/asyncpg-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b6fde867a74e8c76c71e2f64f80c64c0f3163e687f1763cfaf21633ec24ec33", size = 3512103, upload-time = "2024-10-20T00:29:58.499Z" }, - { url = "https://files.pythonhosted.org/packages/96/19/139227a6e67f407b9c386cb594d9628c6c78c9024f26df87c912fabd4368/asyncpg-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46973045b567972128a27d40001124fbc821c87a6cade040cfcd4fa8a30bcdc4", size = 3592471, upload-time = "2024-10-20T00:30:00.354Z" }, - { url = "https://files.pythonhosted.org/packages/67/e4/ab3ca38f628f53f0fd28d3ff20edff1c975dd1cb22482e0061916b4b9a74/asyncpg-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9110df111cabc2ed81aad2f35394a00cadf4f2e0635603db6ebbd0fc896f46a4", size = 3496253, upload-time = "2024-10-20T00:30:02.794Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5f/0bf65511d4eeac3a1f41c54034a492515a707c6edbc642174ae79034d3ba/asyncpg-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04ff0785ae7eed6cc138e73fc67b8e51d54ee7a3ce9b63666ce55a0bf095f7ba", size = 3662720, upload-time = "2024-10-20T00:30:04.501Z" }, - { url = "https://files.pythonhosted.org/packages/e7/31/1513d5a6412b98052c3ed9158d783b1e09d0910f51fbe0e05f56cc370bc4/asyncpg-0.30.0-cp313-cp313-win32.whl", hash = "sha256:ae374585f51c2b444510cdf3595b97ece4f233fde739aa14b50e0d64e8a7a590", size = 560404, upload-time = "2024-10-20T00:30:06.537Z" }, - { url = "https://files.pythonhosted.org/packages/c8/a4/cec76b3389c4c5ff66301cd100fe88c318563ec8a520e0b2e792b5b84972/asyncpg-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:f59b430b8e27557c3fb9869222559f7417ced18688375825f8f12302c34e915e", size = 621623, upload-time = "2024-10-20T00:30:09.024Z" }, -] - [[package]] name = "attrs" version = "25.3.0" @@ -238,17 +151,18 @@ wheels = [ [[package]] name = "backend" -version = "0.1.0" +version = "0.6.0" source = { virtual = "." } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, + { name = "boto3" }, { name = "docker" }, { name = "fastapi" }, { name = "fastmcp" }, - { name = "prefect" }, { name = "pydantic" }, { name = "pyyaml" }, + { name = "temporalio" }, { name = "uvicorn" }, ] @@ -257,32 +171,62 @@ dev = [ { name = "httpx" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-benchmark" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-xdist" }, + { name = "ruff" }, ] [package.metadata] requires-dist = [ { name = "aiofiles", specifier = ">=23.0.0" }, { name = "aiohttp", specifier = ">=3.12.15" }, + { name = "boto3", specifier = ">=1.34.0" }, { name = "docker", specifier = ">=7.0.0" }, { name = "fastapi", specifier = ">=0.116.1" }, { name = "fastmcp" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, - { name = "prefect", specifier = ">=3.4.18" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, + { name = "pytest-benchmark", marker = "extra == 'dev'", specifier = ">=4.0.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.12.0" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, + { name = "temporalio", specifier = ">=1.6.0" }, { name = "uvicorn", specifier = ">=0.30.0" }, ] provides-extras = ["dev"] [[package]] -name = "cachetools" -version = "6.2.0" +name = "boto3" +version = "1.40.44" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/61/e4fad8155db4a04bfb4734c7c8ff0882f078f24294d42798b3568eb63bff/cachetools-6.2.0.tar.gz", hash = "sha256:38b328c0889450f05f5e120f56ab68c8abaf424e1275522b138ffc93253f7e32", size = 30988, upload-time = "2025-08-25T18:57:30.924Z" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/e2/c291748090a9715cc8b74a58e3ba1d17b571b9c1ff6681cfb3191e9c117a/boto3-1.40.44.tar.gz", hash = "sha256:84ade2a253e5445902d2cb2064f48aedf9ba83d6f863244266c2e36c2f190cec", size = 111603, upload-time = "2025-10-02T20:14:25.087Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/56/3124f61d37a7a4e7cc96afc5492c78ba0cb551151e530b54669ddd1436ef/cachetools-6.2.0-py3-none-any.whl", hash = "sha256:1c76a8960c0041fcc21097e357f882197c79da0dbff766e7317890a65d7d8ba6", size = 11276, upload-time = "2025-08-25T18:57:29.684Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3e/3505fd6d192dfc6bbeac09576ba4dbd7d242e9850c275ab0c066433955b7/boto3-1.40.44-py3-none-any.whl", hash = "sha256:281ddf688951773a98161ccb34c54c6376b2ecc7028ab99d77483df5990b448c", size = 139344, upload-time = "2025-10-02T20:14:23.109Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.44" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/97/04/8e4dbfc2ff0ffb0df68de687402a770bbf5c8578e37757d5edacdec5d190/botocore-1.40.44.tar.gz", hash = "sha256:8f6f96ef053dcdfe79c14dfee303c0d381608c111696862fafc6e38402ccf8fe", size = 14391194, upload-time = "2025-10-02T20:14:11.799Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/92/a175f6c442005ed6335592539aa675f2ba0e8478d941186242af742bd912/botocore-1.40.44-py3-none-any.whl", hash = "sha256:6fa7274cdb69be7c7b3ce6ff46a7c3e35e270f259dd77ee3f8ad8c584352262b", size = 14060101, upload-time = "2025-10-02T20:14:08.471Z" }, ] [[package]] @@ -429,15 +373,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -448,12 +383,95 @@ wheels = [ ] [[package]] -name = "coolname" -version = "2.2.0" +name = "coverage" +version = "7.10.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c5/c6/1eaa4495ff4640e80d9af64f540e427ba1596a20f735d4c4750fe0386d07/coolname-2.2.0.tar.gz", hash = "sha256:6c5d5731759104479e7ca195a9b64f7900ac5bead40183c09323c7d0be9e75c7", size = 59006, upload-time = "2023-01-09T14:50:41.724Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/26/d22c300112504f5f9a9fd2297ce33c35f3d353e4aeb987c8419453b2a7c2/coverage-7.10.7.tar.gz", hash = "sha256:f4ab143ab113be368a3e9b795f9cd7906c5ef407d6173fe9675a902e1fffc239", size = 827704, upload-time = "2025-09-21T20:03:56.815Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/b1/5745d7523d8ce53b87779f46ef6cf5c5c342997939c2fe967e607b944e43/coolname-2.2.0-py2.py3-none-any.whl", hash = "sha256:4d1563186cfaf71b394d5df4c744f8c41303b6846413645e31d31915cdeb13e8", size = 37849, upload-time = "2023-01-09T14:50:39.897Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5d/c1a17867b0456f2e9ce2d8d4708a4c3a089947d0bec9c66cdf60c9e7739f/coverage-7.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a609f9c93113be646f44c2a0256d6ea375ad047005d7f57a5c15f614dc1b2f59", size = 218102, upload-time = "2025-09-21T20:01:16.089Z" }, + { url = "https://files.pythonhosted.org/packages/54/f0/514dcf4b4e3698b9a9077f084429681bf3aad2b4a72578f89d7f643eb506/coverage-7.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:65646bb0359386e07639c367a22cf9b5bf6304e8630b565d0626e2bdf329227a", size = 218505, upload-time = "2025-09-21T20:01:17.788Z" }, + { url = "https://files.pythonhosted.org/packages/20/f6/9626b81d17e2a4b25c63ac1b425ff307ecdeef03d67c9a147673ae40dc36/coverage-7.10.7-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5f33166f0dfcce728191f520bd2692914ec70fac2713f6bf3ce59c3deacb4699", size = 248898, upload-time = "2025-09-21T20:01:19.488Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ef/bd8e719c2f7417ba03239052e099b76ea1130ac0cbb183ee1fcaa58aaff3/coverage-7.10.7-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f5e3f9e455bb17831876048355dca0f758b6df22f49258cb5a91da23ef437d", size = 250831, upload-time = "2025-09-21T20:01:20.817Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b6/bf054de41ec948b151ae2b79a55c107f5760979538f5fb80c195f2517718/coverage-7.10.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4da86b6d62a496e908ac2898243920c7992499c1712ff7c2b6d837cc69d9467e", size = 252937, upload-time = "2025-09-21T20:01:22.171Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e5/3860756aa6f9318227443c6ce4ed7bf9e70bb7f1447a0353f45ac5c7974b/coverage-7.10.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6b8b09c1fad947c84bbbc95eca841350fad9cbfa5a2d7ca88ac9f8d836c92e23", size = 249021, upload-time = "2025-09-21T20:01:23.907Z" }, + { url = "https://files.pythonhosted.org/packages/26/0f/bd08bd042854f7fd07b45808927ebcce99a7ed0f2f412d11629883517ac2/coverage-7.10.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4376538f36b533b46f8971d3a3e63464f2c7905c9800db97361c43a2b14792ab", size = 250626, upload-time = "2025-09-21T20:01:25.721Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a7/4777b14de4abcc2e80c6b1d430f5d51eb18ed1d75fca56cbce5f2db9b36e/coverage-7.10.7-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:121da30abb574f6ce6ae09840dae322bef734480ceafe410117627aa54f76d82", size = 248682, upload-time = "2025-09-21T20:01:27.105Z" }, + { url = "https://files.pythonhosted.org/packages/34/72/17d082b00b53cd45679bad682fac058b87f011fd8b9fe31d77f5f8d3a4e4/coverage-7.10.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:88127d40df529336a9836870436fc2751c339fbaed3a836d42c93f3e4bd1d0a2", size = 248402, upload-time = "2025-09-21T20:01:28.629Z" }, + { url = "https://files.pythonhosted.org/packages/81/7a/92367572eb5bdd6a84bfa278cc7e97db192f9f45b28c94a9ca1a921c3577/coverage-7.10.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ba58bbcd1b72f136080c0bccc2400d66cc6115f3f906c499013d065ac33a4b61", size = 249320, upload-time = "2025-09-21T20:01:30.004Z" }, + { url = "https://files.pythonhosted.org/packages/2f/88/a23cc185f6a805dfc4fdf14a94016835eeb85e22ac3a0e66d5e89acd6462/coverage-7.10.7-cp311-cp311-win32.whl", hash = "sha256:972b9e3a4094b053a4e46832b4bc829fc8a8d347160eb39d03f1690316a99c14", size = 220536, upload-time = "2025-09-21T20:01:32.184Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ef/0b510a399dfca17cec7bc2f05ad8bd78cf55f15c8bc9a73ab20c5c913c2e/coverage-7.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:a7b55a944a7f43892e28ad4bc0561dfd5f0d73e605d1aa5c3c976b52aea121d2", size = 221425, upload-time = "2025-09-21T20:01:33.557Z" }, + { url = "https://files.pythonhosted.org/packages/51/7f/023657f301a276e4ba1850f82749bc136f5a7e8768060c2e5d9744a22951/coverage-7.10.7-cp311-cp311-win_arm64.whl", hash = "sha256:736f227fb490f03c6488f9b6d45855f8e0fd749c007f9303ad30efab0e73c05a", size = 220103, upload-time = "2025-09-21T20:01:34.929Z" }, + { url = "https://files.pythonhosted.org/packages/13/e4/eb12450f71b542a53972d19117ea5a5cea1cab3ac9e31b0b5d498df1bd5a/coverage-7.10.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7bb3b9ddb87ef7725056572368040c32775036472d5a033679d1fa6c8dc08417", size = 218290, upload-time = "2025-09-21T20:01:36.455Z" }, + { url = "https://files.pythonhosted.org/packages/37/66/593f9be12fc19fb36711f19a5371af79a718537204d16ea1d36f16bd78d2/coverage-7.10.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:18afb24843cbc175687225cab1138c95d262337f5473512010e46831aa0c2973", size = 218515, upload-time = "2025-09-21T20:01:37.982Z" }, + { url = "https://files.pythonhosted.org/packages/66/80/4c49f7ae09cafdacc73fbc30949ffe77359635c168f4e9ff33c9ebb07838/coverage-7.10.7-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:399a0b6347bcd3822be369392932884b8216d0944049ae22925631a9b3d4ba4c", size = 250020, upload-time = "2025-09-21T20:01:39.617Z" }, + { url = "https://files.pythonhosted.org/packages/a6/90/a64aaacab3b37a17aaedd83e8000142561a29eb262cede42d94a67f7556b/coverage-7.10.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:314f2c326ded3f4b09be11bc282eb2fc861184bc95748ae67b360ac962770be7", size = 252769, upload-time = "2025-09-21T20:01:41.341Z" }, + { url = "https://files.pythonhosted.org/packages/98/2e/2dda59afd6103b342e096f246ebc5f87a3363b5412609946c120f4e7750d/coverage-7.10.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c41e71c9cfb854789dee6fc51e46743a6d138b1803fab6cb860af43265b42ea6", size = 253901, upload-time = "2025-09-21T20:01:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/53/dc/8d8119c9051d50f3119bb4a75f29f1e4a6ab9415cd1fa8bf22fcc3fb3b5f/coverage-7.10.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc01f57ca26269c2c706e838f6422e2a8788e41b3e3c65e2f41148212e57cd59", size = 250413, upload-time = "2025-09-21T20:01:44.469Z" }, + { url = "https://files.pythonhosted.org/packages/98/b3/edaff9c5d79ee4d4b6d3fe046f2b1d799850425695b789d491a64225d493/coverage-7.10.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a6442c59a8ac8b85812ce33bc4d05bde3fb22321fa8294e2a5b487c3505f611b", size = 251820, upload-time = "2025-09-21T20:01:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/11/25/9a0728564bb05863f7e513e5a594fe5ffef091b325437f5430e8cfb0d530/coverage-7.10.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:78a384e49f46b80fb4c901d52d92abe098e78768ed829c673fbb53c498bef73a", size = 249941, upload-time = "2025-09-21T20:01:47.296Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fd/ca2650443bfbef5b0e74373aac4df67b08180d2f184b482c41499668e258/coverage-7.10.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5e1e9802121405ede4b0133aa4340ad8186a1d2526de5b7c3eca519db7bb89fb", size = 249519, upload-time = "2025-09-21T20:01:48.73Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/f692f125fb4299b6f963b0745124998ebb8e73ecdfce4ceceb06a8c6bec5/coverage-7.10.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d41213ea25a86f69efd1575073d34ea11aabe075604ddf3d148ecfec9e1e96a1", size = 251375, upload-time = "2025-09-21T20:01:50.529Z" }, + { url = "https://files.pythonhosted.org/packages/5e/75/61b9bbd6c7d24d896bfeec57acba78e0f8deac68e6baf2d4804f7aae1f88/coverage-7.10.7-cp312-cp312-win32.whl", hash = "sha256:77eb4c747061a6af8d0f7bdb31f1e108d172762ef579166ec84542f711d90256", size = 220699, upload-time = "2025-09-21T20:01:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f3/3bf7905288b45b075918d372498f1cf845b5b579b723c8fd17168018d5f5/coverage-7.10.7-cp312-cp312-win_amd64.whl", hash = "sha256:f51328ffe987aecf6d09f3cd9d979face89a617eacdaea43e7b3080777f647ba", size = 221512, upload-time = "2025-09-21T20:01:53.481Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/3e32dbe933979d05cf2dac5e697c8599cfe038aaf51223ab901e208d5a62/coverage-7.10.7-cp312-cp312-win_arm64.whl", hash = "sha256:bda5e34f8a75721c96085903c6f2197dc398c20ffd98df33f866a9c8fd95f4bf", size = 220147, upload-time = "2025-09-21T20:01:55.2Z" }, + { url = "https://files.pythonhosted.org/packages/9a/94/b765c1abcb613d103b64fcf10395f54d69b0ef8be6a0dd9c524384892cc7/coverage-7.10.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:981a651f543f2854abd3b5fcb3263aac581b18209be49863ba575de6edf4c14d", size = 218320, upload-time = "2025-09-21T20:01:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/72/4f/732fff31c119bb73b35236dd333030f32c4bfe909f445b423e6c7594f9a2/coverage-7.10.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73ab1601f84dc804f7812dc297e93cd99381162da39c47040a827d4e8dafe63b", size = 218575, upload-time = "2025-09-21T20:01:58.203Z" }, + { url = "https://files.pythonhosted.org/packages/87/02/ae7e0af4b674be47566707777db1aa375474f02a1d64b9323e5813a6cdd5/coverage-7.10.7-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a8b6f03672aa6734e700bbcd65ff050fd19cddfec4b031cc8cf1c6967de5a68e", size = 249568, upload-time = "2025-09-21T20:01:59.748Z" }, + { url = "https://files.pythonhosted.org/packages/a2/77/8c6d22bf61921a59bce5471c2f1f7ac30cd4ac50aadde72b8c48d5727902/coverage-7.10.7-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10b6ba00ab1132a0ce4428ff68cf50a25efd6840a42cdf4239c9b99aad83be8b", size = 252174, upload-time = "2025-09-21T20:02:01.192Z" }, + { url = "https://files.pythonhosted.org/packages/b1/20/b6ea4f69bbb52dac0aebd62157ba6a9dddbfe664f5af8122dac296c3ee15/coverage-7.10.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c79124f70465a150e89340de5963f936ee97097d2ef76c869708c4248c63ca49", size = 253447, upload-time = "2025-09-21T20:02:02.701Z" }, + { url = "https://files.pythonhosted.org/packages/f9/28/4831523ba483a7f90f7b259d2018fef02cb4d5b90bc7c1505d6e5a84883c/coverage-7.10.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:69212fbccdbd5b0e39eac4067e20a4a5256609e209547d86f740d68ad4f04911", size = 249779, upload-time = "2025-09-21T20:02:04.185Z" }, + { url = "https://files.pythonhosted.org/packages/a7/9f/4331142bc98c10ca6436d2d620c3e165f31e6c58d43479985afce6f3191c/coverage-7.10.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7ea7c6c9d0d286d04ed3541747e6597cbe4971f22648b68248f7ddcd329207f0", size = 251604, upload-time = "2025-09-21T20:02:06.034Z" }, + { url = "https://files.pythonhosted.org/packages/ce/60/bda83b96602036b77ecf34e6393a3836365481b69f7ed7079ab85048202b/coverage-7.10.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b9be91986841a75042b3e3243d0b3cb0b2434252b977baaf0cd56e960fe1e46f", size = 249497, upload-time = "2025-09-21T20:02:07.619Z" }, + { url = "https://files.pythonhosted.org/packages/5f/af/152633ff35b2af63977edd835d8e6430f0caef27d171edf2fc76c270ef31/coverage-7.10.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b281d5eca50189325cfe1f365fafade89b14b4a78d9b40b05ddd1fc7d2a10a9c", size = 249350, upload-time = "2025-09-21T20:02:10.34Z" }, + { url = "https://files.pythonhosted.org/packages/9d/71/d92105d122bd21cebba877228990e1646d862e34a98bb3374d3fece5a794/coverage-7.10.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:99e4aa63097ab1118e75a848a28e40d68b08a5e19ce587891ab7fd04475e780f", size = 251111, upload-time = "2025-09-21T20:02:12.122Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9e/9fdb08f4bf476c912f0c3ca292e019aab6712c93c9344a1653986c3fd305/coverage-7.10.7-cp313-cp313-win32.whl", hash = "sha256:dc7c389dce432500273eaf48f410b37886be9208b2dd5710aaf7c57fd442c698", size = 220746, upload-time = "2025-09-21T20:02:13.919Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b1/a75fd25df44eab52d1931e89980d1ada46824c7a3210be0d3c88a44aaa99/coverage-7.10.7-cp313-cp313-win_amd64.whl", hash = "sha256:cac0fdca17b036af3881a9d2729a850b76553f3f716ccb0360ad4dbc06b3b843", size = 221541, upload-time = "2025-09-21T20:02:15.57Z" }, + { url = "https://files.pythonhosted.org/packages/14/3a/d720d7c989562a6e9a14b2c9f5f2876bdb38e9367126d118495b89c99c37/coverage-7.10.7-cp313-cp313-win_arm64.whl", hash = "sha256:4b6f236edf6e2f9ae8fcd1332da4e791c1b6ba0dc16a2dc94590ceccb482e546", size = 220170, upload-time = "2025-09-21T20:02:17.395Z" }, + { url = "https://files.pythonhosted.org/packages/bb/22/e04514bf2a735d8b0add31d2b4ab636fc02370730787c576bb995390d2d5/coverage-7.10.7-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a0ec07fd264d0745ee396b666d47cef20875f4ff2375d7c4f58235886cc1ef0c", size = 219029, upload-time = "2025-09-21T20:02:18.936Z" }, + { url = "https://files.pythonhosted.org/packages/11/0b/91128e099035ece15da3445d9015e4b4153a6059403452d324cbb0a575fa/coverage-7.10.7-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd5e856ebb7bfb7672b0086846db5afb4567a7b9714b8a0ebafd211ec7ce6a15", size = 219259, upload-time = "2025-09-21T20:02:20.44Z" }, + { url = "https://files.pythonhosted.org/packages/8b/51/66420081e72801536a091a0c8f8c1f88a5c4bf7b9b1bdc6222c7afe6dc9b/coverage-7.10.7-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f57b2a3c8353d3e04acf75b3fed57ba41f5c0646bbf1d10c7c282291c97936b4", size = 260592, upload-time = "2025-09-21T20:02:22.313Z" }, + { url = "https://files.pythonhosted.org/packages/5d/22/9b8d458c2881b22df3db5bb3e7369e63d527d986decb6c11a591ba2364f7/coverage-7.10.7-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1ef2319dd15a0b009667301a3f84452a4dc6fddfd06b0c5c53ea472d3989fbf0", size = 262768, upload-time = "2025-09-21T20:02:24.287Z" }, + { url = "https://files.pythonhosted.org/packages/f7/08/16bee2c433e60913c610ea200b276e8eeef084b0d200bdcff69920bd5828/coverage-7.10.7-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83082a57783239717ceb0ad584de3c69cf581b2a95ed6bf81ea66034f00401c0", size = 264995, upload-time = "2025-09-21T20:02:26.133Z" }, + { url = "https://files.pythonhosted.org/packages/20/9d/e53eb9771d154859b084b90201e5221bca7674ba449a17c101a5031d4054/coverage-7.10.7-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:50aa94fb1fb9a397eaa19c0d5ec15a5edd03a47bf1a3a6111a16b36e190cff65", size = 259546, upload-time = "2025-09-21T20:02:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b0/69bc7050f8d4e56a89fb550a1577d5d0d1db2278106f6f626464067b3817/coverage-7.10.7-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2120043f147bebb41c85b97ac45dd173595ff14f2a584f2963891cbcc3091541", size = 262544, upload-time = "2025-09-21T20:02:29.216Z" }, + { url = "https://files.pythonhosted.org/packages/ef/4b/2514b060dbd1bc0aaf23b852c14bb5818f244c664cb16517feff6bb3a5ab/coverage-7.10.7-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2fafd773231dd0378fdba66d339f84904a8e57a262f583530f4f156ab83863e6", size = 260308, upload-time = "2025-09-21T20:02:31.226Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/7ba2175007c246d75e496f64c06e94122bdb914790a1285d627a918bd271/coverage-7.10.7-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:0b944ee8459f515f28b851728ad224fa2d068f1513ef6b7ff1efafeb2185f999", size = 258920, upload-time = "2025-09-21T20:02:32.823Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b3/fac9f7abbc841409b9a410309d73bfa6cfb2e51c3fada738cb607ce174f8/coverage-7.10.7-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4b583b97ab2e3efe1b3e75248a9b333bd3f8b0b1b8e5b45578e05e5850dfb2c2", size = 261434, upload-time = "2025-09-21T20:02:34.86Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/a03bec00d37faaa891b3ff7387192cef20f01604e5283a5fabc95346befa/coverage-7.10.7-cp313-cp313t-win32.whl", hash = "sha256:2a78cd46550081a7909b3329e2266204d584866e8d97b898cd7fb5ac8d888b1a", size = 221403, upload-time = "2025-09-21T20:02:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/53/22/3cf25d614e64bf6d8e59c7c669b20d6d940bb337bdee5900b9ca41c820bb/coverage-7.10.7-cp313-cp313t-win_amd64.whl", hash = "sha256:33a5e6396ab684cb43dc7befa386258acb2d7fae7f67330ebb85ba4ea27938eb", size = 222469, upload-time = "2025-09-21T20:02:39.011Z" }, + { url = "https://files.pythonhosted.org/packages/49/a1/00164f6d30d8a01c3c9c48418a7a5be394de5349b421b9ee019f380df2a0/coverage-7.10.7-cp313-cp313t-win_arm64.whl", hash = "sha256:86b0e7308289ddde73d863b7683f596d8d21c7d8664ce1dee061d0bcf3fbb4bb", size = 220731, upload-time = "2025-09-21T20:02:40.939Z" }, + { url = "https://files.pythonhosted.org/packages/23/9c/5844ab4ca6a4dd97a1850e030a15ec7d292b5c5cb93082979225126e35dd/coverage-7.10.7-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b06f260b16ead11643a5a9f955bd4b5fd76c1a4c6796aeade8520095b75de520", size = 218302, upload-time = "2025-09-21T20:02:42.527Z" }, + { url = "https://files.pythonhosted.org/packages/f0/89/673f6514b0961d1f0e20ddc242e9342f6da21eaba3489901b565c0689f34/coverage-7.10.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:212f8f2e0612778f09c55dd4872cb1f64a1f2b074393d139278ce902064d5b32", size = 218578, upload-time = "2025-09-21T20:02:44.468Z" }, + { url = "https://files.pythonhosted.org/packages/05/e8/261cae479e85232828fb17ad536765c88dd818c8470aca690b0ac6feeaa3/coverage-7.10.7-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3445258bcded7d4aa630ab8296dea4d3f15a255588dd535f980c193ab6b95f3f", size = 249629, upload-time = "2025-09-21T20:02:46.503Z" }, + { url = "https://files.pythonhosted.org/packages/82/62/14ed6546d0207e6eda876434e3e8475a3e9adbe32110ce896c9e0c06bb9a/coverage-7.10.7-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb45474711ba385c46a0bfe696c695a929ae69ac636cda8f532be9e8c93d720a", size = 252162, upload-time = "2025-09-21T20:02:48.689Z" }, + { url = "https://files.pythonhosted.org/packages/ff/49/07f00db9ac6478e4358165a08fb41b469a1b053212e8a00cb02f0d27a05f/coverage-7.10.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:813922f35bd800dca9994c5971883cbc0d291128a5de6b167c7aa697fcf59360", size = 253517, upload-time = "2025-09-21T20:02:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/a2/59/c5201c62dbf165dfbc91460f6dbbaa85a8b82cfa6131ac45d6c1bfb52deb/coverage-7.10.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:93c1b03552081b2a4423091d6fb3787265b8f86af404cff98d1b5342713bdd69", size = 249632, upload-time = "2025-09-21T20:02:51.971Z" }, + { url = "https://files.pythonhosted.org/packages/07/ae/5920097195291a51fb00b3a70b9bbd2edbfe3c84876a1762bd1ef1565ebc/coverage-7.10.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cc87dd1b6eaf0b848eebb1c86469b9f72a1891cb42ac7adcfbce75eadb13dd14", size = 251520, upload-time = "2025-09-21T20:02:53.858Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3c/a815dde77a2981f5743a60b63df31cb322c944843e57dbd579326625a413/coverage-7.10.7-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:39508ffda4f343c35f3236fe8d1a6634a51f4581226a1262769d7f970e73bffe", size = 249455, upload-time = "2025-09-21T20:02:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/aa/99/f5cdd8421ea656abefb6c0ce92556709db2265c41e8f9fc6c8ae0f7824c9/coverage-7.10.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:925a1edf3d810537c5a3abe78ec5530160c5f9a26b1f4270b40e62cc79304a1e", size = 249287, upload-time = "2025-09-21T20:02:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/c3/7a/e9a2da6a1fc5d007dd51fca083a663ab930a8c4d149c087732a5dbaa0029/coverage-7.10.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2c8b9a0636f94c43cd3576811e05b89aa9bc2d0a85137affc544ae5cb0e4bfbd", size = 250946, upload-time = "2025-09-21T20:02:59.431Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5b/0b5799aa30380a949005a353715095d6d1da81927d6dbed5def2200a4e25/coverage-7.10.7-cp314-cp314-win32.whl", hash = "sha256:b7b8288eb7cdd268b0304632da8cb0bb93fadcfec2fe5712f7b9cc8f4d487be2", size = 221009, upload-time = "2025-09-21T20:03:01.324Z" }, + { url = "https://files.pythonhosted.org/packages/da/b0/e802fbb6eb746de006490abc9bb554b708918b6774b722bb3a0e6aa1b7de/coverage-7.10.7-cp314-cp314-win_amd64.whl", hash = "sha256:1ca6db7c8807fb9e755d0379ccc39017ce0a84dcd26d14b5a03b78563776f681", size = 221804, upload-time = "2025-09-21T20:03:03.4Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e8/71d0c8e374e31f39e3389bb0bd19e527d46f00ea8571ec7ec8fd261d8b44/coverage-7.10.7-cp314-cp314-win_arm64.whl", hash = "sha256:097c1591f5af4496226d5783d036bf6fd6cd0cbc132e071b33861de756efb880", size = 220384, upload-time = "2025-09-21T20:03:05.111Z" }, + { url = "https://files.pythonhosted.org/packages/62/09/9a5608d319fa3eba7a2019addeacb8c746fb50872b57a724c9f79f146969/coverage-7.10.7-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:a62c6ef0d50e6de320c270ff91d9dd0a05e7250cac2a800b7784bae474506e63", size = 219047, upload-time = "2025-09-21T20:03:06.795Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6f/f58d46f33db9f2e3647b2d0764704548c184e6f5e014bef528b7f979ef84/coverage-7.10.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9fa6e4dd51fe15d8738708a973470f67a855ca50002294852e9571cdbd9433f2", size = 219266, upload-time = "2025-09-21T20:03:08.495Z" }, + { url = "https://files.pythonhosted.org/packages/74/5c/183ffc817ba68e0b443b8c934c8795553eb0c14573813415bd59941ee165/coverage-7.10.7-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8fb190658865565c549b6b4706856d6a7b09302c797eb2cf8e7fe9dabb043f0d", size = 260767, upload-time = "2025-09-21T20:03:10.172Z" }, + { url = "https://files.pythonhosted.org/packages/0f/48/71a8abe9c1ad7e97548835e3cc1adbf361e743e9d60310c5f75c9e7bf847/coverage-7.10.7-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:affef7c76a9ef259187ef31599a9260330e0335a3011732c4b9effa01e1cd6e0", size = 262931, upload-time = "2025-09-21T20:03:11.861Z" }, + { url = "https://files.pythonhosted.org/packages/84/fd/193a8fb132acfc0a901f72020e54be5e48021e1575bb327d8ee1097a28fd/coverage-7.10.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e16e07d85ca0cf8bafe5f5d23a0b850064e8e945d5677492b06bbe6f09cc699", size = 265186, upload-time = "2025-09-21T20:03:13.539Z" }, + { url = "https://files.pythonhosted.org/packages/b1/8f/74ecc30607dd95ad50e3034221113ccb1c6d4e8085cc761134782995daae/coverage-7.10.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:03ffc58aacdf65d2a82bbeb1ffe4d01ead4017a21bfd0454983b88ca73af94b9", size = 259470, upload-time = "2025-09-21T20:03:15.584Z" }, + { url = "https://files.pythonhosted.org/packages/0f/55/79ff53a769f20d71b07023ea115c9167c0bb56f281320520cf64c5298a96/coverage-7.10.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1b4fd784344d4e52647fd7857b2af5b3fbe6c239b0b5fa63e94eb67320770e0f", size = 262626, upload-time = "2025-09-21T20:03:17.673Z" }, + { url = "https://files.pythonhosted.org/packages/88/e2/dac66c140009b61ac3fc13af673a574b00c16efdf04f9b5c740703e953c0/coverage-7.10.7-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0ebbaddb2c19b71912c6f2518e791aa8b9f054985a0769bdb3a53ebbc765c6a1", size = 260386, upload-time = "2025-09-21T20:03:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f1/f48f645e3f33bb9ca8a496bc4a9671b52f2f353146233ebd7c1df6160440/coverage-7.10.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a2d9a3b260cc1d1dbdb1c582e63ddcf5363426a1a68faa0f5da28d8ee3c722a0", size = 258852, upload-time = "2025-09-21T20:03:21.007Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3b/8442618972c51a7affeead957995cfa8323c0c9bcf8fa5a027421f720ff4/coverage-7.10.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a3cc8638b2480865eaa3926d192e64ce6c51e3d29c849e09d5b4ad95efae5399", size = 261534, upload-time = "2025-09-21T20:03:23.12Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dc/101f3fa3a45146db0cb03f5b4376e24c0aac818309da23e2de0c75295a91/coverage-7.10.7-cp314-cp314t-win32.whl", hash = "sha256:67f8c5cbcd3deb7a60b3345dffc89a961a484ed0af1f6f73de91705cc6e31235", size = 221784, upload-time = "2025-09-21T20:03:24.769Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a1/74c51803fc70a8a40d7346660379e144be772bab4ac7bb6e6b905152345c/coverage-7.10.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e1ed71194ef6dea7ed2d5cb5f7243d4bcd334bfb63e59878519be558078f848d", size = 222905, upload-time = "2025-09-21T20:03:26.93Z" }, + { url = "https://files.pythonhosted.org/packages/12/65/f116a6d2127df30bcafbceef0302d8a64ba87488bf6f73a6d8eebf060873/coverage-7.10.7-cp314-cp314t-win_arm64.whl", hash = "sha256:7fe650342addd8524ca63d77b2362b02345e5f1a093266787d210c70a50b471a", size = 220922, upload-time = "2025-09-21T20:03:28.672Z" }, + { url = "https://files.pythonhosted.org/packages/ec/16/114df1c291c22cac3b0c127a73e0af5c12ed7bbb6558d310429a0ae24023/coverage-7.10.7-py3-none-any.whl", hash = "sha256:f7941f6f2fe6dd6807a1208737b8a0cbcf1cc6d7b07d24998ad2d63590868260", size = 209952, upload-time = "2025-09-21T20:03:53.918Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, ] [[package]] @@ -512,21 +530,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f0/8b/2c95f0645c6f40211896375e6fa51f504b8ccb29c21f6ae661fe87ab044e/cyclopts-3.24.0-py3-none-any.whl", hash = "sha256:809d04cde9108617106091140c3964ee6fceb33cecdd537f7ffa360bde13ed71", size = 86154, upload-time = "2025-09-08T15:40:56.41Z" }, ] -[[package]] -name = "dateparser" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "regex" }, - { name = "tzlocal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/30/064144f0df1749e7bb5faaa7f52b007d7c2d08ec08fed8411aba87207f68/dateparser-1.2.2.tar.gz", hash = "sha256:986316f17cb8cdc23ea8ce563027c5ef12fc725b6fb1d137c14ca08777c5ecf7", size = 329840, upload-time = "2025-06-26T09:29:23.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/22/f020c047ae1346613db9322638186468238bcfa8849b4668a22b97faad65/dateparser-1.2.2-py3-none-any.whl", hash = "sha256:5a5d7211a09013499867547023a2a0c91d5a27d15dd4dbcea676ea9fe66f2482", size = 315453, upload-time = "2025-06-26T09:29:21.412Z" }, -] - [[package]] name = "dnspython" version = "2.8.0" @@ -593,6 +596,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "execnet" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/ff/b4c0dc78fbe20c3e59c0c7334de0c27eb4001a2b2017999af398bf730817/execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3", size = 166524, upload-time = "2024-04-08T09:04:19.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/09/2aea36ff60d16dd8879bdb2f5b3ee0ba8d08cbbdcdfe870e695ce3784385/execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc", size = 40612, upload-time = "2024-04-08T09:04:17.414Z" }, +] + [[package]] name = "fastapi" version = "0.116.1" @@ -706,78 +718,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, ] -[[package]] -name = "fsspec" -version = "2025.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847, upload-time = "2025-09-02T19:10:49.215Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, -] - -[[package]] -name = "graphviz" -version = "0.21" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, -] - -[[package]] -name = "greenlet" -version = "3.2.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, - { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, - { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, - { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, - { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, - { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, - { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, - { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, - { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, - { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, - { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, - { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, - { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, - { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, - { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, - { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, - { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, - { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, - { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, - { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, - { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, - { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, - { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, - { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, - { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, - { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, - { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, - { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, - { url = "https://files.pythonhosted.org/packages/c0/aa/687d6b12ffb505a4447567d1f3abea23bd20e73a5bed63871178e0831b7a/greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5", size = 699218, upload-time = "2025-08-07T13:45:30.969Z" }, - { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, - { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, - { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, -] - -[[package]] -name = "griffe" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ec/d7/6c09dd7ce4c7837e4cdb11dce980cb45ae3cd87677298dc3b781b6bce7d3/griffe-1.14.0.tar.gz", hash = "sha256:9d2a15c1eca966d68e00517de5d69dd1bc5c9f2335ef6c1775362ba5b8651a13", size = 424684, upload-time = "2025-09-05T15:02:29.167Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl", hash = "sha256:0e9d52832cccf0f7188cfe585ba962d2674b241c01916d780925df34873bceb0", size = 144439, upload-time = "2025-09-05T15:02:27.511Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -787,28 +727,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] -[[package]] -name = "h2" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "hpack" }, - { name = "hyperframe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, -] - -[[package]] -name = "hpack" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" @@ -837,11 +755,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] -[package.optional-dependencies] -http2 = [ - { name = "h2" }, -] - [[package]] name = "httpx-sse" version = "0.4.1" @@ -851,24 +764,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/25/0a/6269e3473b09aed2dab8aa1a600c70f31f00ae1349bee30658f7e358a159/httpx_sse-0.4.1-py3-none-any.whl", hash = "sha256:cba42174344c3a5b06f255ce65b350880f962d99ead85e776f23c6618a377a37", size = 8054, upload-time = "2025-06-24T13:21:04.772Z" }, ] -[[package]] -name = "humanize" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/1d/3062fcc89ee05a715c0b9bfe6490c00c576314f27ffee3a704122c6fd259/humanize-4.13.0.tar.gz", hash = "sha256:78f79e68f76f0b04d711c4e55d32bebef5be387148862cb1ef83d2b58e7935a0", size = 81884, upload-time = "2025-08-25T09:39:20.04Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/c7/316e7ca04d26695ef0635dc81683d628350810eb8e9b2299fc08ba49f366/humanize-4.13.0-py3-none-any.whl", hash = "sha256:b810820b31891813b1673e8fec7f1ed3312061eab2f26e3fa192c393d11ed25f", size = 128869, upload-time = "2025-08-25T09:39:18.54Z" }, -] - -[[package]] -name = "hyperframe" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, -] - [[package]] name = "idna" version = "3.10" @@ -878,18 +773,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] -[[package]] -name = "importlib-metadata" -version = "8.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, -] - [[package]] name = "iniconfig" version = "2.1.0" @@ -909,49 +792,12 @@ wheels = [ ] [[package]] -name = "jinja2" -version = "3.1.6" +name = "jmespath" +version = "1.0.1" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "jinja2-humanize-extension" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "humanize" }, - { name = "jinja2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/77/0bba383819dd4e67566487c11c49479ced87e77c3285d8e7f7a3401cf882/jinja2_humanize_extension-0.4.0.tar.gz", hash = "sha256:e7d69b1c20f32815bbec722330ee8af14b1287bb1c2b0afa590dbf031cadeaa0", size = 4746, upload-time = "2023-09-01T12:52:42.781Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl", hash = "sha256:b6326e2da0f7d425338bebf58848e830421defbce785f12ae812e65128518156", size = 4769, upload-time = "2023-09-01T12:52:41.098Z" }, -] - -[[package]] -name = "jsonpatch" -version = "1.33" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jsonpointer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699, upload-time = "2023-06-26T12:07:29.144Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, -] - -[[package]] -name = "jsonpointer" -version = "3.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/0a/eebeb1fa92507ea94016a2a790b93c2ae41a7e18778f85471dc54475ed25/jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef", size = 9114, upload-time = "2024-06-10T19:24:42.462Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942", size = 7595, upload-time = "2024-06-10T19:24:40.698Z" }, + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, ] [[package]] @@ -1035,27 +881,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/a0/b91504515c1f9a299fc157967ffbd2f0321bce0516a3d5b89f6f4cad0355/lazy_object_proxy-1.12.0-pp39.pp310.pp311.graalpy311-none-any.whl", hash = "sha256:c3b2e0af1f7f77c4263759c4824316ce458fabe0fceadcd24ef8ca08b2d1e402", size = 15072, upload-time = "2025-08-22T13:50:05.498Z" }, ] -[[package]] -name = "mako" -version = "1.3.10" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, -] - -[[package]] -name = "markdown" -version = "3.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1238,12 +1063,15 @@ wheels = [ ] [[package]] -name = "oauthlib" -version = "3.3.1" +name = "nexus-rpc" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/66/540687556bd28cf1ec370cc6881456203dfddb9dab047b8979c6865b5984/nexus_rpc-1.1.0.tar.gz", hash = "sha256:d65ad6a2f54f14e53ebe39ee30555eaeb894102437125733fb13034a04a44553", size = 77383, upload-time = "2025-07-07T19:03:58.368Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2f/9e9d0dcaa4c6ffa22b7aa31069a8a264c753ff8027b36af602cce038c92f/nexus_rpc-1.1.0-py3-none-any.whl", hash = "sha256:d1b007af2aba186a27e736f8eaae39c03aed05b488084ff6c3d1785c9ba2ad38", size = 27743, upload-time = "2025-07-07T19:03:57.556Z" }, ] [[package]] @@ -1307,83 +1135,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/dd/b3fd642260cb17532f66cc1e8250f3507d1e580483e209dc1e9d13bd980d/openapi_spec_validator-0.7.2-py3-none-any.whl", hash = "sha256:4bbdc0894ec85f1d1bea1d6d9c8b2c3c8d7ccaa13577ef40da9c006c9fd0eb60", size = 39713, upload-time = "2025-06-07T14:48:54.077Z" }, ] -[[package]] -name = "opentelemetry-api" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/04/05040d7ce33a907a2a02257e601992f0cdf11c73b33f13c4492bf6c3d6d5/opentelemetry_api-1.37.0.tar.gz", hash = "sha256:540735b120355bd5112738ea53621f8d5edb35ebcd6fe21ada3ab1c61d1cd9a7", size = 64923, upload-time = "2025-09-11T10:29:01.662Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/48/28ed9e55dcf2f453128df738210a980e09f4e468a456fa3c763dbc8be70a/opentelemetry_api-1.37.0-py3-none-any.whl", hash = "sha256:accf2024d3e89faec14302213bc39550ec0f4095d1cf5ca688e1bfb1c8612f47", size = 65732, upload-time = "2025-09-11T10:28:41.826Z" }, -] - -[[package]] -name = "orjson" -version = "3.11.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/4d/8df5f83256a809c22c4d6792ce8d43bb503be0fb7a8e4da9025754b09658/orjson-3.11.3.tar.gz", hash = "sha256:1c0603b1d2ffcd43a411d64797a19556ef76958aef1c182f22dc30860152a98a", size = 5482394, upload-time = "2025-08-26T17:46:43.171Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/8b/360674cd817faef32e49276187922a946468579fcaf37afdfb6c07046e92/orjson-3.11.3-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9d2ae0cc6aeb669633e0124531f342a17d8e97ea999e42f12a5ad4adaa304c5f", size = 238238, upload-time = "2025-08-26T17:44:54.214Z" }, - { url = "https://files.pythonhosted.org/packages/05/3d/5fa9ea4b34c1a13be7d9046ba98d06e6feb1d8853718992954ab59d16625/orjson-3.11.3-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:ba21dbb2493e9c653eaffdc38819b004b7b1b246fb77bfc93dc016fe664eac91", size = 127713, upload-time = "2025-08-26T17:44:55.596Z" }, - { url = "https://files.pythonhosted.org/packages/e5/5f/e18367823925e00b1feec867ff5f040055892fc474bf5f7875649ecfa586/orjson-3.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f1a271e56d511d1569937c0447d7dce5a99a33ea0dec76673706360a051904", size = 123241, upload-time = "2025-08-26T17:44:57.185Z" }, - { url = "https://files.pythonhosted.org/packages/0f/bd/3c66b91c4564759cf9f473251ac1650e446c7ba92a7c0f9f56ed54f9f0e6/orjson-3.11.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b67e71e47caa6680d1b6f075a396d04fa6ca8ca09aafb428731da9b3ea32a5a6", size = 127895, upload-time = "2025-08-26T17:44:58.349Z" }, - { url = "https://files.pythonhosted.org/packages/82/b5/dc8dcd609db4766e2967a85f63296c59d4722b39503e5b0bf7fd340d387f/orjson-3.11.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7d012ebddffcce8c85734a6d9e5f08180cd3857c5f5a3ac70185b43775d043d", size = 130303, upload-time = "2025-08-26T17:44:59.491Z" }, - { url = "https://files.pythonhosted.org/packages/48/c2/d58ec5fd1270b2aa44c862171891adc2e1241bd7dab26c8f46eb97c6c6f1/orjson-3.11.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd759f75d6b8d1b62012b7f5ef9461d03c804f94d539a5515b454ba3a6588038", size = 132366, upload-time = "2025-08-26T17:45:00.654Z" }, - { url = "https://files.pythonhosted.org/packages/73/87/0ef7e22eb8dd1ef940bfe3b9e441db519e692d62ed1aae365406a16d23d0/orjson-3.11.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6890ace0809627b0dff19cfad92d69d0fa3f089d3e359a2a532507bb6ba34efb", size = 135180, upload-time = "2025-08-26T17:45:02.424Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6a/e5bf7b70883f374710ad74faf99bacfc4b5b5a7797c1d5e130350e0e28a3/orjson-3.11.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d4a5e041ae435b815e568537755773d05dac031fee6a57b4ba70897a44d9d2", size = 132741, upload-time = "2025-08-26T17:45:03.663Z" }, - { url = "https://files.pythonhosted.org/packages/bd/0c/4577fd860b6386ffaa56440e792af01c7882b56d2766f55384b5b0e9d39b/orjson-3.11.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d68bf97a771836687107abfca089743885fb664b90138d8761cce61d5625d55", size = 131104, upload-time = "2025-08-26T17:45:04.939Z" }, - { url = "https://files.pythonhosted.org/packages/66/4b/83e92b2d67e86d1c33f2ea9411742a714a26de63641b082bdbf3d8e481af/orjson-3.11.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:bfc27516ec46f4520b18ef645864cee168d2a027dbf32c5537cb1f3e3c22dac1", size = 403887, upload-time = "2025-08-26T17:45:06.228Z" }, - { url = "https://files.pythonhosted.org/packages/6d/e5/9eea6a14e9b5ceb4a271a1fd2e1dec5f2f686755c0fab6673dc6ff3433f4/orjson-3.11.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f66b001332a017d7945e177e282a40b6997056394e3ed7ddb41fb1813b83e824", size = 145855, upload-time = "2025-08-26T17:45:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/45/78/8d4f5ad0c80ba9bf8ac4d0fc71f93a7d0dc0844989e645e2074af376c307/orjson-3.11.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:212e67806525d2561efbfe9e799633b17eb668b8964abed6b5319b2f1cfbae1f", size = 135361, upload-time = "2025-08-26T17:45:09.625Z" }, - { url = "https://files.pythonhosted.org/packages/0b/5f/16386970370178d7a9b438517ea3d704efcf163d286422bae3b37b88dbb5/orjson-3.11.3-cp311-cp311-win32.whl", hash = "sha256:6e8e0c3b85575a32f2ffa59de455f85ce002b8bdc0662d6b9c2ed6d80ab5d204", size = 136190, upload-time = "2025-08-26T17:45:10.962Z" }, - { url = "https://files.pythonhosted.org/packages/09/60/db16c6f7a41dd8ac9fb651f66701ff2aeb499ad9ebc15853a26c7c152448/orjson-3.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:6be2f1b5d3dc99a5ce5ce162fc741c22ba9f3443d3dd586e6a1211b7bc87bc7b", size = 131389, upload-time = "2025-08-26T17:45:12.285Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2a/bb811ad336667041dea9b8565c7c9faf2f59b47eb5ab680315eea612ef2e/orjson-3.11.3-cp311-cp311-win_arm64.whl", hash = "sha256:fafb1a99d740523d964b15c8db4eabbfc86ff29f84898262bf6e3e4c9e97e43e", size = 126120, upload-time = "2025-08-26T17:45:13.515Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b0/a7edab2a00cdcb2688e1c943401cb3236323e7bfd2839815c6131a3742f4/orjson-3.11.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8c752089db84333e36d754c4baf19c0e1437012242048439c7e80eb0e6426e3b", size = 238259, upload-time = "2025-08-26T17:45:15.093Z" }, - { url = "https://files.pythonhosted.org/packages/e1/c6/ff4865a9cc398a07a83342713b5932e4dc3cb4bf4bc04e8f83dedfc0d736/orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:9b8761b6cf04a856eb544acdd82fc594b978f12ac3602d6374a7edb9d86fd2c2", size = 127633, upload-time = "2025-08-26T17:45:16.417Z" }, - { url = "https://files.pythonhosted.org/packages/6e/e6/e00bea2d9472f44fe8794f523e548ce0ad51eb9693cf538a753a27b8bda4/orjson-3.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b13974dc8ac6ba22feaa867fc19135a3e01a134b4f7c9c28162fed4d615008a", size = 123061, upload-time = "2025-08-26T17:45:17.673Z" }, - { url = "https://files.pythonhosted.org/packages/54/31/9fbb78b8e1eb3ac605467cb846e1c08d0588506028b37f4ee21f978a51d4/orjson-3.11.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f83abab5bacb76d9c821fd5c07728ff224ed0e52d7a71b7b3de822f3df04e15c", size = 127956, upload-time = "2025-08-26T17:45:19.172Z" }, - { url = "https://files.pythonhosted.org/packages/36/88/b0604c22af1eed9f98d709a96302006915cfd724a7ebd27d6dd11c22d80b/orjson-3.11.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6fbaf48a744b94091a56c62897b27c31ee2da93d826aa5b207131a1e13d4064", size = 130790, upload-time = "2025-08-26T17:45:20.586Z" }, - { url = "https://files.pythonhosted.org/packages/0e/9d/1c1238ae9fffbfed51ba1e507731b3faaf6b846126a47e9649222b0fd06f/orjson-3.11.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc779b4f4bba2847d0d2940081a7b6f7b5877e05408ffbb74fa1faf4a136c424", size = 132385, upload-time = "2025-08-26T17:45:22.036Z" }, - { url = "https://files.pythonhosted.org/packages/a3/b5/c06f1b090a1c875f337e21dd71943bc9d84087f7cdf8c6e9086902c34e42/orjson-3.11.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd4b909ce4c50faa2192da6bb684d9848d4510b736b0611b6ab4020ea6fd2d23", size = 135305, upload-time = "2025-08-26T17:45:23.4Z" }, - { url = "https://files.pythonhosted.org/packages/a0/26/5f028c7d81ad2ebbf84414ba6d6c9cac03f22f5cd0d01eb40fb2d6a06b07/orjson-3.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:524b765ad888dc5518bbce12c77c2e83dee1ed6b0992c1790cc5fb49bb4b6667", size = 132875, upload-time = "2025-08-26T17:45:25.182Z" }, - { url = "https://files.pythonhosted.org/packages/fe/d4/b8df70d9cfb56e385bf39b4e915298f9ae6c61454c8154a0f5fd7efcd42e/orjson-3.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:84fd82870b97ae3cdcea9d8746e592b6d40e1e4d4527835fc520c588d2ded04f", size = 130940, upload-time = "2025-08-26T17:45:27.209Z" }, - { url = "https://files.pythonhosted.org/packages/da/5e/afe6a052ebc1a4741c792dd96e9f65bf3939d2094e8b356503b68d48f9f5/orjson-3.11.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbecb9709111be913ae6879b07bafd4b0785b44c1eb5cac8ac76da048b3885a1", size = 403852, upload-time = "2025-08-26T17:45:28.478Z" }, - { url = "https://files.pythonhosted.org/packages/f8/90/7bbabafeb2ce65915e9247f14a56b29c9334003536009ef5b122783fe67e/orjson-3.11.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9dba358d55aee552bd868de348f4736ca5a4086d9a62e2bfbbeeb5629fe8b0cc", size = 146293, upload-time = "2025-08-26T17:45:29.86Z" }, - { url = "https://files.pythonhosted.org/packages/27/b3/2d703946447da8b093350570644a663df69448c9d9330e5f1d9cce997f20/orjson-3.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eabcf2e84f1d7105f84580e03012270c7e97ecb1fb1618bda395061b2a84a049", size = 135470, upload-time = "2025-08-26T17:45:31.243Z" }, - { url = "https://files.pythonhosted.org/packages/38/70/b14dcfae7aff0e379b0119c8a812f8396678919c431efccc8e8a0263e4d9/orjson-3.11.3-cp312-cp312-win32.whl", hash = "sha256:3782d2c60b8116772aea8d9b7905221437fdf53e7277282e8d8b07c220f96cca", size = 136248, upload-time = "2025-08-26T17:45:32.567Z" }, - { url = "https://files.pythonhosted.org/packages/35/b8/9e3127d65de7fff243f7f3e53f59a531bf6bb295ebe5db024c2503cc0726/orjson-3.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:79b44319268af2eaa3e315b92298de9a0067ade6e6003ddaef72f8e0bedb94f1", size = 131437, upload-time = "2025-08-26T17:45:34.949Z" }, - { url = "https://files.pythonhosted.org/packages/51/92/a946e737d4d8a7fd84a606aba96220043dcc7d6988b9e7551f7f6d5ba5ad/orjson-3.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:0e92a4e83341ef79d835ca21b8bd13e27c859e4e9e4d7b63defc6e58462a3710", size = 125978, upload-time = "2025-08-26T17:45:36.422Z" }, - { url = "https://files.pythonhosted.org/packages/fc/79/8932b27293ad35919571f77cb3693b5906cf14f206ef17546052a241fdf6/orjson-3.11.3-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:af40c6612fd2a4b00de648aa26d18186cd1322330bd3a3cc52f87c699e995810", size = 238127, upload-time = "2025-08-26T17:45:38.146Z" }, - { url = "https://files.pythonhosted.org/packages/1c/82/cb93cd8cf132cd7643b30b6c5a56a26c4e780c7a145db6f83de977b540ce/orjson-3.11.3-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:9f1587f26c235894c09e8b5b7636a38091a9e6e7fe4531937534749c04face43", size = 127494, upload-time = "2025-08-26T17:45:39.57Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b8/2d9eb181a9b6bb71463a78882bcac1027fd29cf62c38a40cc02fc11d3495/orjson-3.11.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61dcdad16da5bb486d7227a37a2e789c429397793a6955227cedbd7252eb5a27", size = 123017, upload-time = "2025-08-26T17:45:40.876Z" }, - { url = "https://files.pythonhosted.org/packages/b4/14/a0e971e72d03b509190232356d54c0f34507a05050bd026b8db2bf2c192c/orjson-3.11.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:11c6d71478e2cbea0a709e8a06365fa63da81da6498a53e4c4f065881d21ae8f", size = 127898, upload-time = "2025-08-26T17:45:42.188Z" }, - { url = "https://files.pythonhosted.org/packages/8e/af/dc74536722b03d65e17042cc30ae586161093e5b1f29bccda24765a6ae47/orjson-3.11.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff94112e0098470b665cb0ed06efb187154b63649403b8d5e9aedeb482b4548c", size = 130742, upload-time = "2025-08-26T17:45:43.511Z" }, - { url = "https://files.pythonhosted.org/packages/62/e6/7a3b63b6677bce089fe939353cda24a7679825c43a24e49f757805fc0d8a/orjson-3.11.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae8b756575aaa2a855a75192f356bbda11a89169830e1439cfb1a3e1a6dde7be", size = 132377, upload-time = "2025-08-26T17:45:45.525Z" }, - { url = "https://files.pythonhosted.org/packages/fc/cd/ce2ab93e2e7eaf518f0fd15e3068b8c43216c8a44ed82ac2b79ce5cef72d/orjson-3.11.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9416cc19a349c167ef76135b2fe40d03cea93680428efee8771f3e9fb66079d", size = 135313, upload-time = "2025-08-26T17:45:46.821Z" }, - { url = "https://files.pythonhosted.org/packages/d0/b4/f98355eff0bd1a38454209bbc73372ce351ba29933cb3e2eba16c04b9448/orjson-3.11.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b822caf5b9752bc6f246eb08124c3d12bf2175b66ab74bac2ef3bbf9221ce1b2", size = 132908, upload-time = "2025-08-26T17:45:48.126Z" }, - { url = "https://files.pythonhosted.org/packages/eb/92/8f5182d7bc2a1bed46ed960b61a39af8389f0ad476120cd99e67182bfb6d/orjson-3.11.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:414f71e3bdd5573893bf5ecdf35c32b213ed20aa15536fe2f588f946c318824f", size = 130905, upload-time = "2025-08-26T17:45:49.414Z" }, - { url = "https://files.pythonhosted.org/packages/1a/60/c41ca753ce9ffe3d0f67b9b4c093bdd6e5fdb1bc53064f992f66bb99954d/orjson-3.11.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:828e3149ad8815dc14468f36ab2a4b819237c155ee1370341b91ea4c8672d2ee", size = 403812, upload-time = "2025-08-26T17:45:51.085Z" }, - { url = "https://files.pythonhosted.org/packages/dd/13/e4a4f16d71ce1868860db59092e78782c67082a8f1dc06a3788aef2b41bc/orjson-3.11.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ac9e05f25627ffc714c21f8dfe3a579445a5c392a9c8ae7ba1d0e9fb5333f56e", size = 146277, upload-time = "2025-08-26T17:45:52.851Z" }, - { url = "https://files.pythonhosted.org/packages/8d/8b/bafb7f0afef9344754a3a0597a12442f1b85a048b82108ef2c956f53babd/orjson-3.11.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e44fbe4000bd321d9f3b648ae46e0196d21577cf66ae684a96ff90b1f7c93633", size = 135418, upload-time = "2025-08-26T17:45:54.806Z" }, - { url = "https://files.pythonhosted.org/packages/60/d4/bae8e4f26afb2c23bea69d2f6d566132584d1c3a5fe89ee8c17b718cab67/orjson-3.11.3-cp313-cp313-win32.whl", hash = "sha256:2039b7847ba3eec1f5886e75e6763a16e18c68a63efc4b029ddf994821e2e66b", size = 136216, upload-time = "2025-08-26T17:45:57.182Z" }, - { url = "https://files.pythonhosted.org/packages/88/76/224985d9f127e121c8cad882cea55f0ebe39f97925de040b75ccd4b33999/orjson-3.11.3-cp313-cp313-win_amd64.whl", hash = "sha256:29be5ac4164aa8bdcba5fa0700a3c9c316b411d8ed9d39ef8a882541bd452fae", size = 131362, upload-time = "2025-08-26T17:45:58.56Z" }, - { url = "https://files.pythonhosted.org/packages/e2/cf/0dce7a0be94bd36d1346be5067ed65ded6adb795fdbe3abd234c8d576d01/orjson-3.11.3-cp313-cp313-win_arm64.whl", hash = "sha256:18bd1435cb1f2857ceb59cfb7de6f92593ef7b831ccd1b9bfb28ca530e539dce", size = 125989, upload-time = "2025-08-26T17:45:59.95Z" }, - { url = "https://files.pythonhosted.org/packages/ef/77/d3b1fef1fc6aaeed4cbf3be2b480114035f4df8fa1a99d2dac1d40d6e924/orjson-3.11.3-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cf4b81227ec86935568c7edd78352a92e97af8da7bd70bdfdaa0d2e0011a1ab4", size = 238115, upload-time = "2025-08-26T17:46:01.669Z" }, - { url = "https://files.pythonhosted.org/packages/e4/6d/468d21d49bb12f900052edcfbf52c292022d0a323d7828dc6376e6319703/orjson-3.11.3-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:bc8bc85b81b6ac9fc4dae393a8c159b817f4c2c9dee5d12b773bddb3b95fc07e", size = 127493, upload-time = "2025-08-26T17:46:03.466Z" }, - { url = "https://files.pythonhosted.org/packages/67/46/1e2588700d354aacdf9e12cc2d98131fb8ac6f31ca65997bef3863edb8ff/orjson-3.11.3-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:88dcfc514cfd1b0de038443c7b3e6a9797ffb1b3674ef1fd14f701a13397f82d", size = 122998, upload-time = "2025-08-26T17:46:04.803Z" }, - { url = "https://files.pythonhosted.org/packages/3b/94/11137c9b6adb3779f1b34fd98be51608a14b430dbc02c6d41134fbba484c/orjson-3.11.3-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:d61cd543d69715d5fc0a690c7c6f8dcc307bc23abef9738957981885f5f38229", size = 132915, upload-time = "2025-08-26T17:46:06.237Z" }, - { url = "https://files.pythonhosted.org/packages/10/61/dccedcf9e9bcaac09fdabe9eaee0311ca92115699500efbd31950d878833/orjson-3.11.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2b7b153ed90ababadbef5c3eb39549f9476890d339cf47af563aea7e07db2451", size = 130907, upload-time = "2025-08-26T17:46:07.581Z" }, - { url = "https://files.pythonhosted.org/packages/0e/fd/0e935539aa7b08b3ca0f817d73034f7eb506792aae5ecc3b7c6e679cdf5f/orjson-3.11.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7909ae2460f5f494fecbcd10613beafe40381fd0316e35d6acb5f3a05bfda167", size = 403852, upload-time = "2025-08-26T17:46:08.982Z" }, - { url = "https://files.pythonhosted.org/packages/4a/2b/50ae1a5505cd1043379132fdb2adb8a05f37b3e1ebffe94a5073321966fd/orjson-3.11.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2030c01cbf77bc67bee7eef1e7e31ecf28649353987775e3583062c752da0077", size = 146309, upload-time = "2025-08-26T17:46:10.576Z" }, - { url = "https://files.pythonhosted.org/packages/cd/1d/a473c158e380ef6f32753b5f39a69028b25ec5be331c2049a2201bde2e19/orjson-3.11.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0169ebd1cbd94b26c7a7ad282cf5c2744fce054133f959e02eb5265deae1872", size = 135424, upload-time = "2025-08-26T17:46:12.386Z" }, - { url = "https://files.pythonhosted.org/packages/da/09/17d9d2b60592890ff7382e591aa1d9afb202a266b180c3d4049b1ec70e4a/orjson-3.11.3-cp314-cp314-win32.whl", hash = "sha256:0c6d7328c200c349e3a4c6d8c83e0a5ad029bdc2d417f234152bf34842d0fc8d", size = 136266, upload-time = "2025-08-26T17:46:13.853Z" }, - { url = "https://files.pythonhosted.org/packages/15/58/358f6846410a6b4958b74734727e582ed971e13d335d6c7ce3e47730493e/orjson-3.11.3-cp314-cp314-win_amd64.whl", hash = "sha256:317bbe2c069bbc757b1a2e4105b64aacd3bc78279b66a6b9e51e846e4809f804", size = 131351, upload-time = "2025-08-26T17:46:15.27Z" }, - { url = "https://files.pythonhosted.org/packages/28/01/d6b274a0635be0468d4dbd9cafe80c47105937a0d42434e805e67cd2ed8b/orjson-3.11.3-cp314-cp314-win_arm64.whl", hash = "sha256:e8f6a7a27d7b7bec81bd5924163e9af03d49bbb63013f107b48eb5d16db711bc", size = 125985, upload-time = "2025-08-26T17:46:16.67Z" }, -] - [[package]] name = "packaging" version = "25.0" @@ -1411,58 +1162,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/eb/b6260b31b1a96386c0a880edebe26f89669098acea8e0318bff6adb378fd/pathable-0.4.4-py3-none-any.whl", hash = "sha256:5ae9e94793b6ef5a4cbe0a7ce9dbbefc1eec38df253763fd0aeeacf2762dbbc2", size = 9592, upload-time = "2025-01-10T18:43:11.88Z" }, ] -[[package]] -name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, -] - -[[package]] -name = "pendulum" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/7c/009c12b86c7cc6c403aec80f8a4308598dfc5995e5c523a5491faaa3952e/pendulum-3.1.0.tar.gz", hash = "sha256:66f96303560f41d097bee7d2dc98ffca716fbb3a832c4b3062034c2d45865015", size = 85930, upload-time = "2025-04-19T14:30:01.675Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/6e/d28d3c22e6708b819a94c05bd05a3dfaed5c685379e8b6dc4b34b473b942/pendulum-3.1.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:61a03d14f8c64d13b2f7d5859e4b4053c4a7d3b02339f6c71f3e4606bfd67423", size = 338596, upload-time = "2025-04-19T14:01:11.306Z" }, - { url = "https://files.pythonhosted.org/packages/e1/e6/43324d58021d463c2eeb6146b169d2c935f2f840f9e45ac2d500453d954c/pendulum-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e674ed2d158afa5c361e60f1f67872dc55b492a10cacdaa7fcd7b7da5f158f24", size = 325854, upload-time = "2025-04-19T14:01:13.156Z" }, - { url = "https://files.pythonhosted.org/packages/b0/a7/d2ae79b960bfdea94dab67e2f118697b08bc9e98eb6bd8d32c4d99240da3/pendulum-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c75377eb16e58bbe7e03ea89eeea49be6fc5de0934a4aef0e263f8b4fa71bc2", size = 344334, upload-time = "2025-04-19T14:01:15.151Z" }, - { url = "https://files.pythonhosted.org/packages/96/94/941f071212e23c29aae7def891fb636930c648386e059ce09ea0dcd43933/pendulum-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:656b8b0ce070f0f2e5e2668247d3c783c55336534aa1f13bd0969535878955e1", size = 382259, upload-time = "2025-04-19T14:01:16.924Z" }, - { url = "https://files.pythonhosted.org/packages/51/ad/a78a701656aec00d16fee636704445c23ca11617a0bfe7c3848d1caa5157/pendulum-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48962903e6c1afe1f13548cb6252666056086c107d59e3d64795c58c9298bc2e", size = 436361, upload-time = "2025-04-19T14:01:18.796Z" }, - { url = "https://files.pythonhosted.org/packages/da/93/83f59ccbf4435c29dca8c63a6560fcbe4783079a468a5f91d9f886fd21f0/pendulum-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d364ec3f8e65010fefd4b0aaf7be5eb97e5df761b107a06f5e743b7c3f52c311", size = 353653, upload-time = "2025-04-19T14:01:20.159Z" }, - { url = "https://files.pythonhosted.org/packages/6f/0f/42d6644ec6339b41066f594e52d286162aecd2e9735aaf994d7e00c9e09d/pendulum-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dd52caffc2afb86612ec43bbeb226f204ea12ebff9f3d12f900a7d3097210fcc", size = 524567, upload-time = "2025-04-19T14:01:21.457Z" }, - { url = "https://files.pythonhosted.org/packages/de/45/d84d909202755ab9d3379e5481fdf70f53344ebefbd68d6f5803ddde98a6/pendulum-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d439fccaa35c91f686bd59d30604dab01e8b5c1d0dd66e81648c432fd3f8a539", size = 525571, upload-time = "2025-04-19T14:01:23.329Z" }, - { url = "https://files.pythonhosted.org/packages/0d/e0/4de160773ce3c2f7843c310db19dd919a0cd02cc1c0384866f63b18a6251/pendulum-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:43288773a86d9c5c0ddb645f88f615ff6bd12fd1410b34323662beccb18f3b49", size = 260259, upload-time = "2025-04-19T14:01:24.689Z" }, - { url = "https://files.pythonhosted.org/packages/c1/7f/ffa278f78112c6c6e5130a702042f52aab5c649ae2edf814df07810bbba5/pendulum-3.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:569ea5072ae0f11d625e03b36d865f8037b76e838a3b621f6967314193896a11", size = 253899, upload-time = "2025-04-19T14:01:26.442Z" }, - { url = "https://files.pythonhosted.org/packages/7a/d7/b1bfe15a742f2c2713acb1fdc7dc3594ff46ef9418ac6a96fcb12a6ba60b/pendulum-3.1.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4dfd53e7583ccae138be86d6c0a0b324c7547df2afcec1876943c4d481cf9608", size = 336209, upload-time = "2025-04-19T14:01:27.815Z" }, - { url = "https://files.pythonhosted.org/packages/eb/87/0392da0c603c828b926d9f7097fbdddaafc01388cb8a00888635d04758c3/pendulum-3.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a6e06a28f3a7d696546347805536f6f38be458cb79de4f80754430696bea9e6", size = 323130, upload-time = "2025-04-19T14:01:29.336Z" }, - { url = "https://files.pythonhosted.org/packages/c0/61/95f1eec25796be6dddf71440ee16ec1fd0c573fc61a73bd1ef6daacd529a/pendulum-3.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e68d6a51880708084afd8958af42dc8c5e819a70a6c6ae903b1c4bfc61e0f25", size = 341509, upload-time = "2025-04-19T14:01:31.1Z" }, - { url = "https://files.pythonhosted.org/packages/b5/7b/eb0f5e6aa87d5e1b467a1611009dbdc92f0f72425ebf07669bfadd8885a6/pendulum-3.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e3f1e5da39a7ea7119efda1dd96b529748c1566f8a983412d0908455d606942", size = 378674, upload-time = "2025-04-19T14:01:32.974Z" }, - { url = "https://files.pythonhosted.org/packages/29/68/5a4c1b5de3e54e16cab21d2ec88f9cd3f18599e96cc90a441c0b0ab6b03f/pendulum-3.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9af1e5eeddb4ebbe1b1c9afb9fd8077d73416ade42dd61264b3f3b87742e0bb", size = 436133, upload-time = "2025-04-19T14:01:34.349Z" }, - { url = "https://files.pythonhosted.org/packages/87/5d/f7a1d693e5c0f789185117d5c1d5bee104f5b0d9fbf061d715fb61c840a8/pendulum-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20f74aa8029a42e327bfc150472e0e4d2358fa5d795f70460160ba81b94b6945", size = 351232, upload-time = "2025-04-19T14:01:35.669Z" }, - { url = "https://files.pythonhosted.org/packages/30/77/c97617eb31f1d0554edb073201a294019b9e0a9bd2f73c68e6d8d048cd6b/pendulum-3.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:cf6229e5ee70c2660148523f46c472e677654d0097bec010d6730f08312a4931", size = 521562, upload-time = "2025-04-19T14:01:37.05Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/0d0ef3393303877e757b848ecef8a9a8c7627e17e7590af82d14633b2cd1/pendulum-3.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:350cabb23bf1aec7c7694b915d3030bff53a2ad4aeabc8c8c0d807c8194113d6", size = 523221, upload-time = "2025-04-19T14:01:38.444Z" }, - { url = "https://files.pythonhosted.org/packages/99/f3/aefb579aa3cebd6f2866b205fc7a60d33e9a696e9e629024752107dc3cf5/pendulum-3.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:42959341e843077c41d47420f28c3631de054abd64da83f9b956519b5c7a06a7", size = 260502, upload-time = "2025-04-19T14:01:39.814Z" }, - { url = "https://files.pythonhosted.org/packages/02/74/4332b5d6e34c63d4df8e8eab2249e74c05513b1477757463f7fdca99e9be/pendulum-3.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:006758e2125da2e624493324dfd5d7d1b02b0c44bc39358e18bf0f66d0767f5f", size = 253089, upload-time = "2025-04-19T14:01:41.171Z" }, - { url = "https://files.pythonhosted.org/packages/8e/1f/af928ba4aa403dac9569f787adcf024005e7654433d71f7a84e608716837/pendulum-3.1.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:28658b0baf4b30eb31d096a375983cfed033e60c0a7bbe94fa23f06cd779b50b", size = 336209, upload-time = "2025-04-19T14:01:42.775Z" }, - { url = "https://files.pythonhosted.org/packages/b6/16/b010643007ba964c397da7fa622924423883c1bbff1a53f9d1022cd7f024/pendulum-3.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b114dcb99ce511cb8f5495c7b6f0056b2c3dba444ef1ea6e48030d7371bd531a", size = 323132, upload-time = "2025-04-19T14:01:44.577Z" }, - { url = "https://files.pythonhosted.org/packages/64/19/c3c47aeecb5d9bceb0e89faafd800d39809b696c5b7bba8ec8370ad5052c/pendulum-3.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2404a6a54c80252ea393291f0b7f35525a61abae3d795407f34e118a8f133a18", size = 341509, upload-time = "2025-04-19T14:01:46.084Z" }, - { url = "https://files.pythonhosted.org/packages/38/cf/c06921ff6b860ff7e62e70b8e5d4dc70e36f5abb66d168bd64d51760bc4e/pendulum-3.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d06999790d9ee9962a1627e469f98568bf7ad1085553fa3c30ed08b3944a14d7", size = 378674, upload-time = "2025-04-19T14:01:47.727Z" }, - { url = "https://files.pythonhosted.org/packages/62/0b/a43953b9eba11e82612b033ac5133f716f1b76b6108a65da6f408b3cc016/pendulum-3.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94751c52f6b7c306734d1044c2c6067a474237e1e5afa2f665d1fbcbbbcf24b3", size = 436133, upload-time = "2025-04-19T14:01:49.126Z" }, - { url = "https://files.pythonhosted.org/packages/eb/a0/ec3d70b3b96e23ae1d039f132af35e17704c22a8250d1887aaefea4d78a6/pendulum-3.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5553ac27be05e997ec26d7f004cf72788f4ce11fe60bb80dda604a64055b29d0", size = 351232, upload-time = "2025-04-19T14:01:50.575Z" }, - { url = "https://files.pythonhosted.org/packages/f4/97/aba23f1716b82f6951ba2b1c9178a2d107d1e66c102762a9bf19988547ea/pendulum-3.1.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:f8dee234ca6142bf0514368d01a72945a44685aaa2fc4c14c98d09da9437b620", size = 521563, upload-time = "2025-04-19T14:01:51.9Z" }, - { url = "https://files.pythonhosted.org/packages/01/33/2c0d5216cc53d16db0c4b3d510f141ee0a540937f8675948541190fbd48b/pendulum-3.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7378084fe54faab4ee481897a00b710876f2e901ded6221671e827a253e643f2", size = 523221, upload-time = "2025-04-19T14:01:53.275Z" }, - { url = "https://files.pythonhosted.org/packages/51/89/8de955c339c31aeae77fd86d3225509b998c81875e9dba28cb88b8cbf4b3/pendulum-3.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8539db7ae2c8da430ac2515079e288948c8ebf7eb1edd3e8281b5cdf433040d6", size = 260501, upload-time = "2025-04-19T14:01:54.749Z" }, - { url = "https://files.pythonhosted.org/packages/15/c3/226a3837363e94f8722461848feec18bfdd7d5172564d53aa3c3397ff01e/pendulum-3.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:1ce26a608e1f7387cd393fba2a129507c4900958d4f47b90757ec17656856571", size = 253087, upload-time = "2025-04-19T14:01:55.998Z" }, - { url = "https://files.pythonhosted.org/packages/6e/23/e98758924d1b3aac11a626268eabf7f3cf177e7837c28d47bf84c64532d0/pendulum-3.1.0-py3-none-any.whl", hash = "sha256:f9178c2a8e291758ade1e8dd6371b1d26d08371b4c7730a6e9a3ef8b16ebae0f", size = 111799, upload-time = "2025-04-19T14:02:34.739Z" }, -] - [[package]] name = "pluggy" version = "1.6.0" @@ -1472,80 +1171,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "prefect" -version = "3.4.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiosqlite" }, - { name = "alembic" }, - { name = "anyio" }, - { name = "apprise" }, - { name = "asgi-lifespan" }, - { name = "asyncpg" }, - { name = "cachetools" }, - { name = "click" }, - { name = "cloudpickle" }, - { name = "coolname" }, - { name = "cryptography" }, - { name = "dateparser" }, - { name = "docker" }, - { name = "exceptiongroup" }, - { name = "fastapi" }, - { name = "fsspec" }, - { name = "graphviz" }, - { name = "griffe" }, - { name = "httpcore" }, - { name = "httpx", extra = ["http2"] }, - { name = "humanize" }, - { name = "jinja2" }, - { name = "jinja2-humanize-extension" }, - { name = "jsonpatch" }, - { name = "jsonschema" }, - { name = "opentelemetry-api" }, - { name = "orjson" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "pendulum", marker = "python_full_version < '3.13'" }, - { name = "prometheus-client" }, - { name = "pydantic" }, - { name = "pydantic-core" }, - { name = "pydantic-extra-types" }, - { name = "pydantic-settings" }, - { name = "python-dateutil" }, - { name = "python-slugify" }, - { name = "python-socks" }, - { name = "pytz" }, - { name = "pyyaml" }, - { name = "readchar" }, - { name = "rfc3339-validator" }, - { name = "rich" }, - { name = "ruamel-yaml" }, - { name = "semver" }, - { name = "sniffio" }, - { name = "sqlalchemy", extra = ["asyncio"] }, - { name = "toml" }, - { name = "typer" }, - { name = "typing-extensions" }, - { name = "uv" }, - { name = "uvicorn" }, - { name = "websockets" }, - { name = "whenever", marker = "python_full_version >= '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/f6/e581bb9e43b43f212e08879e522a60e785b43b2678037b2e4bc9f8661594/prefect-3.4.18.tar.gz", hash = "sha256:04a5af7b5d0fcd1202315e32d46f6067dcf912bfd58d3ed113c74c0f58abd1cf", size = 5599981, upload-time = "2025-09-12T16:22:02.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/f9/c5b61846b8fb33541c07e08170504c46e44a0ed45ad2b649fa15d7af11ec/prefect-3.4.18-py3-none-any.whl", hash = "sha256:a98824b91eb8de8d4fac84bd1ef08ce64564b1904b89f4c55059d30523b533ed", size = 6112304, upload-time = "2025-09-12T16:21:59.394Z" }, -] - -[[package]] -name = "prometheus-client" -version = "0.22.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5e/cf/40dde0a2be27cc1eb41e333d1a674a74ce8b8b0457269cc640fd42b07cf7/prometheus_client-0.22.1.tar.gz", hash = "sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28", size = 69746, upload-time = "2025-06-02T14:29:01.152Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/ae/ec06af4fe3ee72d16973474f122541746196aaa16cea6f66d18b963c6177/prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094", size = 58694, upload-time = "2025-06-02T14:29:00.068Z" }, -] - [[package]] name = "propcache" version = "0.3.2" @@ -1619,6 +1244,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" }, ] +[[package]] +name = "protobuf" +version = "6.32.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/98/645183ea03ab3995d29086b8bf4f7562ebd3d10c9a4b14ee3f20d47cfe50/protobuf-6.32.1-cp310-abi3-win32.whl", hash = "sha256:a8a32a84bc9f2aad712041b8b366190f71dde248926da517bde9e832e4412085", size = 424411, upload-time = "2025-09-11T21:38:27.427Z" }, + { url = "https://files.pythonhosted.org/packages/8c/f3/6f58f841f6ebafe076cebeae33fc336e900619d34b1c93e4b5c97a81fdfa/protobuf-6.32.1-cp310-abi3-win_amd64.whl", hash = "sha256:b00a7d8c25fa471f16bc8153d0e53d6c9e827f0953f3c09aaa4331c718cae5e1", size = 435738, upload-time = "2025-09-11T21:38:30.959Z" }, + { url = "https://files.pythonhosted.org/packages/10/56/a8a3f4e7190837139e68c7002ec749190a163af3e330f65d90309145a210/protobuf-6.32.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8c7e6eb619ffdf105ee4ab76af5a68b60a9d0f66da3ea12d1640e6d8dab7281", size = 426454, upload-time = "2025-09-11T21:38:34.076Z" }, + { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" }, + { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "pycparser" version = "2.23" @@ -1713,19 +1361,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757, upload-time = "2025-04-23T18:33:30.645Z" }, ] -[[package]] -name = "pydantic-extra-types" -version = "2.10.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/ba/4178111ec4116c54e1dc7ecd2a1ff8f54256cdbd250e576882911e8f710a/pydantic_extra_types-2.10.5.tar.gz", hash = "sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8", size = 138429, upload-time = "2025-06-02T09:31:52.713Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/1a/5f4fd9e7285f10c44095a4f9fe17d0f358d1702a7c74a9278c794e8a7537/pydantic_extra_types-2.10.5-py3-none-any.whl", hash = "sha256:b60c4e23d573a69a4f1a16dd92888ecc0ef34fb0e655b4f305530377fa70e7a8", size = 38315, upload-time = "2025-06-02T09:31:51.229Z" }, -] - [[package]] name = "pydantic-settings" version = "2.10.1" @@ -1787,6 +1422,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/93/2fa34714b7a4ae72f2f8dad66ba17dd9a2c793220719e736dda28b7aec27/pytest_asyncio-1.2.0-py3-none-any.whl", hash = "sha256:8e17ae5e46d8e7efe51ab6494dd2010f4ca8dae51652aa3c8d55acf50bfb2e99", size = 15095, upload-time = "2025-09-12T07:33:52.639Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1817,36 +1504,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, ] -[[package]] -name = "python-slugify" -version = "8.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "text-unidecode" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/c7/5e1547c44e31da50a460df93af11a535ace568ef89d7a811069ead340c4a/python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856", size = 10921, upload-time = "2024-02-08T18:32:45.488Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, -] - -[[package]] -name = "python-socks" -version = "2.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c0/fb/49fc4c3d61dbc8404879bed6c94c0595e654951ac9145645b057c4883966/python_socks-2.7.2.tar.gz", hash = "sha256:4c845d4700352bc7e7382f302dfc6baf0af0de34d2a6d70ba356b2539d4dbb62", size = 229950, upload-time = "2025-08-01T06:47:05.488Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/e6/1fdebffa733e79e67b43ee8930e4e5049eb51eae3608caeafc83518798aa/python_socks-2.7.2-py3-none-any.whl", hash = "sha256:d311aefbacc0ddfaa1fa1c32096c436d4fe75b899c24d78e677e1b0623c52c48", size = 55048, upload-time = "2025-08-01T06:47:03.734Z" }, -] - -[[package]] -name = "pytz" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, -] - [[package]] name = "pywin32" version = "311" @@ -1901,15 +1558,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, ] -[[package]] -name = "readchar" -version = "4.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dd/f8/8657b8cbb4ebeabfbdf991ac40eca8a1d1bd012011bd44ad1ed10f5cb494/readchar-4.2.1.tar.gz", hash = "sha256:91ce3faf07688de14d800592951e5575e9c7a3213738ed01d394dcc949b79adb", size = 9685, upload-time = "2024-11-04T18:28:07.757Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/10/e4b1e0e5b6b6745c8098c275b69bc9d73e9542d5c7da4f137542b499ed44/readchar-4.2.1-py3-none-any.whl", hash = "sha256:a769305cd3994bb5fa2764aa4073452dc105a4ec39068ffe6efd3c20c60acc77", size = 9350, upload-time = "2024-11-04T18:28:02.859Z" }, -] - [[package]] name = "referencing" version = "0.36.2" @@ -1924,70 +1572,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, ] -[[package]] -name = "regex" -version = "2025.9.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/5a/4c63457fbcaf19d138d72b2e9b39405954f98c0349b31c601bfcb151582c/regex-2025.9.1.tar.gz", hash = "sha256:88ac07b38d20b54d79e704e38aa3bd2c0f8027432164226bdee201a1c0c9c9ff", size = 400852, upload-time = "2025-09-01T22:10:10.479Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/4d/f741543c0c59f96c6625bc6c11fea1da2e378b7d293ffff6f318edc0ce14/regex-2025.9.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e5bcf112b09bfd3646e4db6bf2e598534a17d502b0c01ea6550ba4eca780c5e6", size = 484811, upload-time = "2025-09-01T22:08:12.834Z" }, - { url = "https://files.pythonhosted.org/packages/c2/bd/27e73e92635b6fbd51afc26a414a3133243c662949cd1cda677fe7bb09bd/regex-2025.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:67a0295a3c31d675a9ee0238d20238ff10a9a2fdb7a1323c798fc7029578b15c", size = 288977, upload-time = "2025-09-01T22:08:14.499Z" }, - { url = "https://files.pythonhosted.org/packages/eb/7d/7dc0c6efc8bc93cd6e9b947581f5fde8a5dbaa0af7c4ec818c5729fdc807/regex-2025.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea8267fbadc7d4bd7c1301a50e85c2ff0de293ff9452a1a9f8d82c6cafe38179", size = 286606, upload-time = "2025-09-01T22:08:15.881Z" }, - { url = "https://files.pythonhosted.org/packages/d1/01/9b5c6dd394f97c8f2c12f6e8f96879c9ac27292a718903faf2e27a0c09f6/regex-2025.9.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6aeff21de7214d15e928fb5ce757f9495214367ba62875100d4c18d293750cc1", size = 792436, upload-time = "2025-09-01T22:08:17.38Z" }, - { url = "https://files.pythonhosted.org/packages/fc/24/b7430cfc6ee34bbb3db6ff933beb5e7692e5cc81e8f6f4da63d353566fb0/regex-2025.9.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d89f1bbbbbc0885e1c230f7770d5e98f4f00b0ee85688c871d10df8b184a6323", size = 858705, upload-time = "2025-09-01T22:08:19.037Z" }, - { url = "https://files.pythonhosted.org/packages/d6/98/155f914b4ea6ae012663188545c4f5216c11926d09b817127639d618b003/regex-2025.9.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca3affe8ddea498ba9d294ab05f5f2d3b5ad5d515bc0d4a9016dd592a03afe52", size = 905881, upload-time = "2025-09-01T22:08:20.377Z" }, - { url = "https://files.pythonhosted.org/packages/8a/a7/a470e7bc8259c40429afb6d6a517b40c03f2f3e455c44a01abc483a1c512/regex-2025.9.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91892a7a9f0a980e4c2c85dd19bc14de2b219a3a8867c4b5664b9f972dcc0c78", size = 798968, upload-time = "2025-09-01T22:08:22.081Z" }, - { url = "https://files.pythonhosted.org/packages/1d/fa/33f6fec4d41449fea5f62fdf5e46d668a1c046730a7f4ed9f478331a8e3a/regex-2025.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e1cb40406f4ae862710615f9f636c1e030fd6e6abe0e0f65f6a695a2721440c6", size = 781884, upload-time = "2025-09-01T22:08:23.832Z" }, - { url = "https://files.pythonhosted.org/packages/42/de/2b45f36ab20da14eedddf5009d370625bc5942d9953fa7e5037a32d66843/regex-2025.9.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:94f6cff6f7e2149c7e6499a6ecd4695379eeda8ccbccb9726e8149f2fe382e92", size = 852935, upload-time = "2025-09-01T22:08:25.536Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f9/878f4fc92c87e125e27aed0f8ee0d1eced9b541f404b048f66f79914475a/regex-2025.9.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6c0226fb322b82709e78c49cc33484206647f8a39954d7e9de1567f5399becd0", size = 844340, upload-time = "2025-09-01T22:08:27.141Z" }, - { url = "https://files.pythonhosted.org/packages/90/c2/5b6f2bce6ece5f8427c718c085eca0de4bbb4db59f54db77aa6557aef3e9/regex-2025.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a12f59c7c380b4fcf7516e9cbb126f95b7a9518902bcf4a852423ff1dcd03e6a", size = 787238, upload-time = "2025-09-01T22:08:28.75Z" }, - { url = "https://files.pythonhosted.org/packages/47/66/1ef1081c831c5b611f6f55f6302166cfa1bc9574017410ba5595353f846a/regex-2025.9.1-cp311-cp311-win32.whl", hash = "sha256:49865e78d147a7a4f143064488da5d549be6bfc3f2579e5044cac61f5c92edd4", size = 264118, upload-time = "2025-09-01T22:08:30.388Z" }, - { url = "https://files.pythonhosted.org/packages/ad/e0/8adc550d7169df1d6b9be8ff6019cda5291054a0107760c2f30788b6195f/regex-2025.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:d34b901f6f2f02ef60f4ad3855d3a02378c65b094efc4b80388a3aeb700a5de7", size = 276151, upload-time = "2025-09-01T22:08:32.073Z" }, - { url = "https://files.pythonhosted.org/packages/cb/bd/46fef29341396d955066e55384fb93b0be7d64693842bf4a9a398db6e555/regex-2025.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:47d7c2dab7e0b95b95fd580087b6ae196039d62306a592fa4e162e49004b6299", size = 268460, upload-time = "2025-09-01T22:08:33.281Z" }, - { url = "https://files.pythonhosted.org/packages/39/ef/a0372febc5a1d44c1be75f35d7e5aff40c659ecde864d7fa10e138f75e74/regex-2025.9.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:84a25164bd8dcfa9f11c53f561ae9766e506e580b70279d05a7946510bdd6f6a", size = 486317, upload-time = "2025-09-01T22:08:34.529Z" }, - { url = "https://files.pythonhosted.org/packages/b5/25/d64543fb7eb41a1024786d518cc57faf1ce64aa6e9ddba097675a0c2f1d2/regex-2025.9.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:645e88a73861c64c1af558dd12294fb4e67b5c1eae0096a60d7d8a2143a611c7", size = 289698, upload-time = "2025-09-01T22:08:36.162Z" }, - { url = "https://files.pythonhosted.org/packages/d8/dc/fbf31fc60be317bd9f6f87daa40a8a9669b3b392aa8fe4313df0a39d0722/regex-2025.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10a450cba5cd5409526ee1d4449f42aad38dd83ac6948cbd6d7f71ca7018f7db", size = 287242, upload-time = "2025-09-01T22:08:37.794Z" }, - { url = "https://files.pythonhosted.org/packages/0f/74/f933a607a538f785da5021acf5323961b4620972e2c2f1f39b6af4b71db7/regex-2025.9.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9dc5991592933a4192c166eeb67b29d9234f9c86344481173d1bc52f73a7104", size = 797441, upload-time = "2025-09-01T22:08:39.108Z" }, - { url = "https://files.pythonhosted.org/packages/89/d0/71fc49b4f20e31e97f199348b8c4d6e613e7b6a54a90eb1b090c2b8496d7/regex-2025.9.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a32291add816961aab472f4fad344c92871a2ee33c6c219b6598e98c1f0108f2", size = 862654, upload-time = "2025-09-01T22:08:40.586Z" }, - { url = "https://files.pythonhosted.org/packages/59/05/984edce1411a5685ba9abbe10d42cdd9450aab4a022271f9585539788150/regex-2025.9.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:588c161a68a383478e27442a678e3b197b13c5ba51dbba40c1ccb8c4c7bee9e9", size = 910862, upload-time = "2025-09-01T22:08:42.416Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/5c891bb5fe0691cc1bad336e3a94b9097fbcf9707ec8ddc1dce9f0397289/regex-2025.9.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47829ffaf652f30d579534da9085fe30c171fa2a6744a93d52ef7195dc38218b", size = 801991, upload-time = "2025-09-01T22:08:44.072Z" }, - { url = "https://files.pythonhosted.org/packages/f1/ae/fd10d6ad179910f7a1b3e0a7fde1ef8bb65e738e8ac4fd6ecff3f52252e4/regex-2025.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e978e5a35b293ea43f140c92a3269b6ab13fe0a2bf8a881f7ac740f5a6ade85", size = 786651, upload-time = "2025-09-01T22:08:46.079Z" }, - { url = "https://files.pythonhosted.org/packages/30/cf/9d686b07bbc5bf94c879cc168db92542d6bc9fb67088d03479fef09ba9d3/regex-2025.9.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4cf09903e72411f4bf3ac1eddd624ecfd423f14b2e4bf1c8b547b72f248b7bf7", size = 856556, upload-time = "2025-09-01T22:08:48.376Z" }, - { url = "https://files.pythonhosted.org/packages/91/9d/302f8a29bb8a49528abbab2d357a793e2a59b645c54deae0050f8474785b/regex-2025.9.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d016b0f77be63e49613c9e26aaf4a242f196cd3d7a4f15898f5f0ab55c9b24d2", size = 849001, upload-time = "2025-09-01T22:08:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/93/fa/b4c6dbdedc85ef4caec54c817cd5f4418dbfa2453214119f2538082bf666/regex-2025.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:656563e620de6908cd1c9d4f7b9e0777e3341ca7db9d4383bcaa44709c90281e", size = 788138, upload-time = "2025-09-01T22:08:51.933Z" }, - { url = "https://files.pythonhosted.org/packages/4a/1b/91ee17a3cbf87f81e8c110399279d0e57f33405468f6e70809100f2ff7d8/regex-2025.9.1-cp312-cp312-win32.whl", hash = "sha256:df33f4ef07b68f7ab637b1dbd70accbf42ef0021c201660656601e8a9835de45", size = 264524, upload-time = "2025-09-01T22:08:53.75Z" }, - { url = "https://files.pythonhosted.org/packages/92/28/6ba31cce05b0f1ec6b787921903f83bd0acf8efde55219435572af83c350/regex-2025.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5aba22dfbc60cda7c0853516104724dc904caa2db55f2c3e6e984eb858d3edf3", size = 275489, upload-time = "2025-09-01T22:08:55.037Z" }, - { url = "https://files.pythonhosted.org/packages/bd/ed/ea49f324db00196e9ef7fe00dd13c6164d5173dd0f1bbe495e61bb1fb09d/regex-2025.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:ec1efb4c25e1849c2685fa95da44bfde1b28c62d356f9c8d861d4dad89ed56e9", size = 268589, upload-time = "2025-09-01T22:08:56.369Z" }, - { url = "https://files.pythonhosted.org/packages/98/25/b2959ce90c6138c5142fe5264ee1f9b71a0c502ca4c7959302a749407c79/regex-2025.9.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:bc6834727d1b98d710a63e6c823edf6ffbf5792eba35d3fa119531349d4142ef", size = 485932, upload-time = "2025-09-01T22:08:57.913Z" }, - { url = "https://files.pythonhosted.org/packages/49/2e/6507a2a85f3f2be6643438b7bd976e67ad73223692d6988eb1ff444106d3/regex-2025.9.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c3dc05b6d579875719bccc5f3037b4dc80433d64e94681a0061845bd8863c025", size = 289568, upload-time = "2025-09-01T22:08:59.258Z" }, - { url = "https://files.pythonhosted.org/packages/c7/d8/de4a4b57215d99868f1640e062a7907e185ec7476b4b689e2345487c1ff4/regex-2025.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:22213527df4c985ec4a729b055a8306272d41d2f45908d7bacb79be0fa7a75ad", size = 286984, upload-time = "2025-09-01T22:09:00.835Z" }, - { url = "https://files.pythonhosted.org/packages/03/15/e8cb403403a57ed316e80661db0e54d7aa2efcd85cb6156f33cc18746922/regex-2025.9.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e3f6e3c5a5a1adc3f7ea1b5aec89abfc2f4fbfba55dafb4343cd1d084f715b2", size = 797514, upload-time = "2025-09-01T22:09:02.538Z" }, - { url = "https://files.pythonhosted.org/packages/e4/26/2446f2b9585fed61faaa7e2bbce3aca7dd8df6554c32addee4c4caecf24a/regex-2025.9.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcb89c02a0d6c2bec9b0bb2d8c78782699afe8434493bfa6b4021cc51503f249", size = 862586, upload-time = "2025-09-01T22:09:04.322Z" }, - { url = "https://files.pythonhosted.org/packages/fd/b8/82ffbe9c0992c31bbe6ae1c4b4e21269a5df2559102b90543c9b56724c3c/regex-2025.9.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b0e2f95413eb0c651cd1516a670036315b91b71767af83bc8525350d4375ccba", size = 910815, upload-time = "2025-09-01T22:09:05.978Z" }, - { url = "https://files.pythonhosted.org/packages/2f/d8/7303ea38911759c1ee30cc5bc623ee85d3196b733c51fd6703c34290a8d9/regex-2025.9.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09a41dc039e1c97d3c2ed3e26523f748e58c4de3ea7a31f95e1cf9ff973fff5a", size = 802042, upload-time = "2025-09-01T22:09:07.865Z" }, - { url = "https://files.pythonhosted.org/packages/fc/0e/6ad51a55ed4b5af512bb3299a05d33309bda1c1d1e1808fa869a0bed31bc/regex-2025.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f0b4258b161094f66857a26ee938d3fe7b8a5063861e44571215c44fbf0e5df", size = 786764, upload-time = "2025-09-01T22:09:09.362Z" }, - { url = "https://files.pythonhosted.org/packages/8d/d5/394e3ffae6baa5a9217bbd14d96e0e5da47bb069d0dbb8278e2681a2b938/regex-2025.9.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bf70e18ac390e6977ea7e56f921768002cb0fa359c4199606c7219854ae332e0", size = 856557, upload-time = "2025-09-01T22:09:11.129Z" }, - { url = "https://files.pythonhosted.org/packages/cd/80/b288d3910c41194ad081b9fb4b371b76b0bbfdce93e7709fc98df27b37dc/regex-2025.9.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b84036511e1d2bb0a4ff1aec26951caa2dea8772b223c9e8a19ed8885b32dbac", size = 849108, upload-time = "2025-09-01T22:09:12.877Z" }, - { url = "https://files.pythonhosted.org/packages/d1/cd/5ec76bf626d0d5abdc277b7a1734696f5f3d14fbb4a3e2540665bc305d85/regex-2025.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c2e05dcdfe224047f2a59e70408274c325d019aad96227ab959403ba7d58d2d7", size = 788201, upload-time = "2025-09-01T22:09:14.561Z" }, - { url = "https://files.pythonhosted.org/packages/b5/36/674672f3fdead107565a2499f3007788b878188acec6d42bc141c5366c2c/regex-2025.9.1-cp313-cp313-win32.whl", hash = "sha256:3b9a62107a7441b81ca98261808fed30ae36ba06c8b7ee435308806bd53c1ed8", size = 264508, upload-time = "2025-09-01T22:09:16.193Z" }, - { url = "https://files.pythonhosted.org/packages/83/ad/931134539515eb64ce36c24457a98b83c1b2e2d45adf3254b94df3735a76/regex-2025.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:b38afecc10c177eb34cfae68d669d5161880849ba70c05cbfbe409f08cc939d7", size = 275469, upload-time = "2025-09-01T22:09:17.462Z" }, - { url = "https://files.pythonhosted.org/packages/24/8c/96d34e61c0e4e9248836bf86d69cb224fd222f270fa9045b24e218b65604/regex-2025.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:ec329890ad5e7ed9fc292858554d28d58d56bf62cf964faf0aa57964b21155a0", size = 268586, upload-time = "2025-09-01T22:09:18.948Z" }, - { url = "https://files.pythonhosted.org/packages/21/b1/453cbea5323b049181ec6344a803777914074b9726c9c5dc76749966d12d/regex-2025.9.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:72fb7a016467d364546f22b5ae86c45680a4e0de6b2a6f67441d22172ff641f1", size = 486111, upload-time = "2025-09-01T22:09:20.734Z" }, - { url = "https://files.pythonhosted.org/packages/f6/0e/92577f197bd2f7652c5e2857f399936c1876978474ecc5b068c6d8a79c86/regex-2025.9.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:c9527fa74eba53f98ad86be2ba003b3ebe97e94b6eb2b916b31b5f055622ef03", size = 289520, upload-time = "2025-09-01T22:09:22.249Z" }, - { url = "https://files.pythonhosted.org/packages/af/c6/b472398116cca7ea5a6c4d5ccd0fc543f7fd2492cb0c48d2852a11972f73/regex-2025.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c905d925d194c83a63f92422af7544ec188301451b292c8b487f0543726107ca", size = 287215, upload-time = "2025-09-01T22:09:23.657Z" }, - { url = "https://files.pythonhosted.org/packages/cf/11/f12ecb0cf9ca792a32bb92f758589a84149017467a544f2f6bfb45c0356d/regex-2025.9.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74df7c74a63adcad314426b1f4ea6054a5ab25d05b0244f0c07ff9ce640fa597", size = 797855, upload-time = "2025-09-01T22:09:25.197Z" }, - { url = "https://files.pythonhosted.org/packages/46/88/bbb848f719a540fb5997e71310f16f0b33a92c5d4b4d72d4311487fff2a3/regex-2025.9.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4f6e935e98ea48c7a2e8be44494de337b57a204470e7f9c9c42f912c414cd6f5", size = 863363, upload-time = "2025-09-01T22:09:26.705Z" }, - { url = "https://files.pythonhosted.org/packages/54/a9/2321eb3e2838f575a78d48e03c1e83ea61bd08b74b7ebbdeca8abc50fc25/regex-2025.9.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4a62d033cd9ebefc7c5e466731a508dfabee827d80b13f455de68a50d3c2543d", size = 910202, upload-time = "2025-09-01T22:09:28.906Z" }, - { url = "https://files.pythonhosted.org/packages/33/07/d1d70835d7d11b7e126181f316f7213c4572ecf5c5c97bdbb969fb1f38a2/regex-2025.9.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef971ebf2b93bdc88d8337238be4dfb851cc97ed6808eb04870ef67589415171", size = 801808, upload-time = "2025-09-01T22:09:30.733Z" }, - { url = "https://files.pythonhosted.org/packages/13/d1/29e4d1bed514ef2bf3a4ead3cb8bb88ca8af94130239a4e68aa765c35b1c/regex-2025.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d936a1db208bdca0eca1f2bb2c1ba1d8370b226785c1e6db76e32a228ffd0ad5", size = 786824, upload-time = "2025-09-01T22:09:32.61Z" }, - { url = "https://files.pythonhosted.org/packages/33/27/20d8ccb1bee460faaa851e6e7cc4cfe852a42b70caa1dca22721ba19f02f/regex-2025.9.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:7e786d9e4469698fc63815b8de08a89165a0aa851720eb99f5e0ea9d51dd2b6a", size = 857406, upload-time = "2025-09-01T22:09:34.117Z" }, - { url = "https://files.pythonhosted.org/packages/74/fe/60c6132262dc36430d51e0c46c49927d113d3a38c1aba6a26c7744c84cf3/regex-2025.9.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6b81d7dbc5466ad2c57ce3a0ddb717858fe1a29535c8866f8514d785fdb9fc5b", size = 848593, upload-time = "2025-09-01T22:09:35.598Z" }, - { url = "https://files.pythonhosted.org/packages/cc/ae/2d4ff915622fabbef1af28387bf71e7f2f4944a348b8460d061e85e29bf0/regex-2025.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cd4890e184a6feb0ef195338a6ce68906a8903a0f2eb7e0ab727dbc0a3156273", size = 787951, upload-time = "2025-09-01T22:09:37.139Z" }, - { url = "https://files.pythonhosted.org/packages/85/37/dc127703a9e715a284cc2f7dbdd8a9776fd813c85c126eddbcbdd1ca5fec/regex-2025.9.1-cp314-cp314-win32.whl", hash = "sha256:34679a86230e46164c9e0396b56cab13c0505972343880b9e705083cc5b8ec86", size = 269833, upload-time = "2025-09-01T22:09:39.245Z" }, - { url = "https://files.pythonhosted.org/packages/83/bf/4bed4d3d0570e16771defd5f8f15f7ea2311edcbe91077436d6908956c4a/regex-2025.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:a1196e530a6bfa5f4bde029ac5b0295a6ecfaaffbfffede4bbaf4061d9455b70", size = 278742, upload-time = "2025-09-01T22:09:40.651Z" }, - { url = "https://files.pythonhosted.org/packages/cf/3e/7d7ac6fd085023312421e0d69dfabdfb28e116e513fadbe9afe710c01893/regex-2025.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:f46d525934871ea772930e997d577d48c6983e50f206ff7b66d4ac5f8941e993", size = 271860, upload-time = "2025-09-01T22:09:42.413Z" }, -] - [[package]] name = "requests" version = "2.32.5" @@ -2003,19 +1587,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "oauthlib" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, -] - [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -2163,68 +1734,41 @@ wheels = [ ] [[package]] -name = "ruamel-yaml" -version = "0.18.15" +name = "ruff" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/8e/f9f9ca747fea8e3ac954e3690d4698c9737c23b51731d02df999c150b1c9/ruff-0.13.3.tar.gz", hash = "sha256:5b0ba0db740eefdfbcce4299f49e9eaefc643d4d007749d77d047c2bab19908e", size = 5438533, upload-time = "2025-10-02T19:29:31.582Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/33/8f7163553481466a92656d35dea9331095122bb84cf98210bef597dd2ecd/ruff-0.13.3-py3-none-linux_armv6l.whl", hash = "sha256:311860a4c5e19189c89d035638f500c1e191d283d0cc2f1600c8c80d6dcd430c", size = 12484040, upload-time = "2025-10-02T19:28:49.199Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b5/4a21a4922e5dd6845e91896b0d9ef493574cbe061ef7d00a73c61db531af/ruff-0.13.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2bdad6512fb666b40fcadb65e33add2b040fc18a24997d2e47fee7d66f7fcae2", size = 13122975, upload-time = "2025-10-02T19:28:52.446Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/15649af836d88c9f154e5be87e64ae7d2b1baa5a3ef317cb0c8fafcd882d/ruff-0.13.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fc6fa4637284708d6ed4e5e970d52fc3b76a557d7b4e85a53013d9d201d93286", size = 12346621, upload-time = "2025-10-02T19:28:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/bcbccb8141305f9a6d3f72549dd82d1134299177cc7eaf832599700f95a7/ruff-0.13.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c9e6469864f94a98f412f20ea143d547e4c652f45e44f369d7b74ee78185838", size = 12574408, upload-time = "2025-10-02T19:28:56.679Z" }, + { url = "https://files.pythonhosted.org/packages/ce/19/0f3681c941cdcfa2d110ce4515624c07a964dc315d3100d889fcad3bfc9e/ruff-0.13.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5bf62b705f319476c78891e0e97e965b21db468b3c999086de8ffb0d40fd2822", size = 12285330, upload-time = "2025-10-02T19:28:58.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/f8/387976bf00d126b907bbd7725219257feea58650e6b055b29b224d8cb731/ruff-0.13.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cc1abed87ce40cb07ee0667ce99dbc766c9f519eabfd948ed87295d8737c60", size = 13980815, upload-time = "2025-10-02T19:29:01.577Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a6/7c8ec09d62d5a406e2b17d159e4817b63c945a8b9188a771193b7e1cc0b5/ruff-0.13.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4fb75e7c402d504f7a9a259e0442b96403fa4a7310ffe3588d11d7e170d2b1e3", size = 14987733, upload-time = "2025-10-02T19:29:04.036Z" }, + { url = "https://files.pythonhosted.org/packages/97/e5/f403a60a12258e0fd0c2195341cfa170726f254c788673495d86ab5a9a9d/ruff-0.13.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:17b951f9d9afb39330b2bdd2dd144ce1c1335881c277837ac1b50bfd99985ed3", size = 14439848, upload-time = "2025-10-02T19:29:06.684Z" }, + { url = "https://files.pythonhosted.org/packages/39/49/3de381343e89364c2334c9f3268b0349dc734fc18b2d99a302d0935c8345/ruff-0.13.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6052f8088728898e0a449f0dde8fafc7ed47e4d878168b211977e3e7e854f662", size = 13421890, upload-time = "2025-10-02T19:29:08.767Z" }, + { url = "https://files.pythonhosted.org/packages/ab/b5/c0feca27d45ae74185a6bacc399f5d8920ab82df2d732a17213fb86a2c4c/ruff-0.13.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc742c50f4ba72ce2a3be362bd359aef7d0d302bf7637a6f942eaa763bd292af", size = 13444870, upload-time = "2025-10-02T19:29:11.234Z" }, + { url = "https://files.pythonhosted.org/packages/50/a1/b655298a1f3fda4fdc7340c3f671a4b260b009068fbeb3e4e151e9e3e1bf/ruff-0.13.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8e5640349493b378431637019366bbd73c927e515c9c1babfea3e932f5e68e1d", size = 13691599, upload-time = "2025-10-02T19:29:13.353Z" }, + { url = "https://files.pythonhosted.org/packages/32/b0/a8705065b2dafae007bcae21354e6e2e832e03eb077bb6c8e523c2becb92/ruff-0.13.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6b139f638a80eae7073c691a5dd8d581e0ba319540be97c343d60fb12949c8d0", size = 12421893, upload-time = "2025-10-02T19:29:15.668Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1e/cbe7082588d025cddbb2f23e6dfef08b1a2ef6d6f8328584ad3015b5cebd/ruff-0.13.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:6b547def0a40054825de7cfa341039ebdfa51f3d4bfa6a0772940ed351d2746c", size = 12267220, upload-time = "2025-10-02T19:29:17.583Z" }, + { url = "https://files.pythonhosted.org/packages/a5/99/4086f9c43f85e0755996d09bdcb334b6fee9b1eabdf34e7d8b877fadf964/ruff-0.13.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9cc48a3564423915c93573f1981d57d101e617839bef38504f85f3677b3a0a3e", size = 13177818, upload-time = "2025-10-02T19:29:19.943Z" }, + { url = "https://files.pythonhosted.org/packages/9b/de/7b5db7e39947d9dc1c5f9f17b838ad6e680527d45288eeb568e860467010/ruff-0.13.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1a993b17ec03719c502881cb2d5f91771e8742f2ca6de740034433a97c561989", size = 13618715, upload-time = "2025-10-02T19:29:22.527Z" }, + { url = "https://files.pythonhosted.org/packages/28/d3/bb25ee567ce2f61ac52430cf99f446b0e6d49bdfa4188699ad005fdd16aa/ruff-0.13.3-py3-none-win32.whl", hash = "sha256:f14e0d1fe6460f07814d03c6e32e815bff411505178a1f539a38f6097d3e8ee3", size = 12334488, upload-time = "2025-10-02T19:29:24.782Z" }, + { url = "https://files.pythonhosted.org/packages/cf/49/12f5955818a1139eed288753479ba9d996f6ea0b101784bb1fe6977ec128/ruff-0.13.3-py3-none-win_amd64.whl", hash = "sha256:621e2e5812b691d4f244638d693e640f188bacbb9bc793ddd46837cea0503dd2", size = 13455262, upload-time = "2025-10-02T19:29:26.882Z" }, + { url = "https://files.pythonhosted.org/packages/fe/72/7b83242b26627a00e3af70d0394d68f8f02750d642567af12983031777fc/ruff-0.13.3-py3-none-win_arm64.whl", hash = "sha256:9e9e9d699841eaf4c2c798fa783df2fabc680b72059a02ca0ed81c460bc58330", size = 12538484, upload-time = "2025-10-02T19:29:28.951Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ruamel-yaml-clib", marker = "python_full_version < '3.14' and platform_python_implementation == 'CPython'" }, + { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3e/db/f3950f5e5031b618aae9f423a39bf81a55c148aecd15a34527898e752cf4/ruamel.yaml-0.18.15.tar.gz", hash = "sha256:dbfca74b018c4c3fba0b9cc9ee33e53c371194a9000e694995e620490fd40700", size = 146865, upload-time = "2025-08-19T11:15:10.694Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/e5/f2a0621f1781b76a38194acae72f01e37b1941470407345b6e8653ad7640/ruamel.yaml-0.18.15-py3-none-any.whl", hash = "sha256:148f6488d698b7a5eded5ea793a025308b25eca97208181b6a026037f391f701", size = 119702, upload-time = "2025-08-19T11:15:07.696Z" }, -] - -[[package]] -name = "ruamel-yaml-clib" -version = "0.2.12" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/84/80203abff8ea4993a87d823a5f632e4d92831ef75d404c9fc78d0176d2b5/ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f", size = 225315, upload-time = "2024-10-20T10:10:56.22Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/8f/683c6ad562f558cbc4f7c029abcd9599148c51c54b5ef0f24f2638da9fbb/ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6", size = 132224, upload-time = "2024-10-20T10:12:45.162Z" }, - { url = "https://files.pythonhosted.org/packages/3c/d2/b79b7d695e2f21da020bd44c782490578f300dd44f0a4c57a92575758a76/ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e", size = 641480, upload-time = "2024-10-20T10:12:46.758Z" }, - { url = "https://files.pythonhosted.org/packages/68/6e/264c50ce2a31473a9fdbf4fa66ca9b2b17c7455b31ef585462343818bd6c/ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e", size = 739068, upload-time = "2024-10-20T10:12:48.605Z" }, - { url = "https://files.pythonhosted.org/packages/86/29/88c2567bc893c84d88b4c48027367c3562ae69121d568e8a3f3a8d363f4d/ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52", size = 703012, upload-time = "2024-10-20T10:12:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/11/46/879763c619b5470820f0cd6ca97d134771e502776bc2b844d2adb6e37753/ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642", size = 704352, upload-time = "2024-10-21T11:26:41.438Z" }, - { url = "https://files.pythonhosted.org/packages/02/80/ece7e6034256a4186bbe50dee28cd032d816974941a6abf6a9d65e4228a7/ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2", size = 737344, upload-time = "2024-10-21T11:26:43.62Z" }, - { url = "https://files.pythonhosted.org/packages/f0/ca/e4106ac7e80efbabdf4bf91d3d32fc424e41418458251712f5672eada9ce/ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1492a6051dab8d912fc2adeef0e8c72216b24d57bd896ea607cb90bb0c4981d3", size = 714498, upload-time = "2024-12-11T19:58:15.592Z" }, - { url = "https://files.pythonhosted.org/packages/67/58/b1f60a1d591b771298ffa0428237afb092c7f29ae23bad93420b1eb10703/ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4", size = 100205, upload-time = "2024-10-20T10:12:52.865Z" }, - { url = "https://files.pythonhosted.org/packages/b4/4f/b52f634c9548a9291a70dfce26ca7ebce388235c93588a1068028ea23fcc/ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb", size = 118185, upload-time = "2024-10-20T10:12:54.652Z" }, - { url = "https://files.pythonhosted.org/packages/48/41/e7a405afbdc26af961678474a55373e1b323605a4f5e2ddd4a80ea80f628/ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632", size = 133433, upload-time = "2024-10-20T10:12:55.657Z" }, - { url = "https://files.pythonhosted.org/packages/ec/b0/b850385604334c2ce90e3ee1013bd911aedf058a934905863a6ea95e9eb4/ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d", size = 647362, upload-time = "2024-10-20T10:12:57.155Z" }, - { url = "https://files.pythonhosted.org/packages/44/d0/3f68a86e006448fb6c005aee66565b9eb89014a70c491d70c08de597f8e4/ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c", size = 754118, upload-time = "2024-10-20T10:12:58.501Z" }, - { url = "https://files.pythonhosted.org/packages/52/a9/d39f3c5ada0a3bb2870d7db41901125dbe2434fa4f12ca8c5b83a42d7c53/ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd", size = 706497, upload-time = "2024-10-20T10:13:00.211Z" }, - { url = "https://files.pythonhosted.org/packages/b0/fa/097e38135dadd9ac25aecf2a54be17ddf6e4c23e43d538492a90ab3d71c6/ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31", size = 698042, upload-time = "2024-10-21T11:26:46.038Z" }, - { url = "https://files.pythonhosted.org/packages/ec/d5/a659ca6f503b9379b930f13bc6b130c9f176469b73b9834296822a83a132/ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680", size = 745831, upload-time = "2024-10-21T11:26:47.487Z" }, - { url = "https://files.pythonhosted.org/packages/db/5d/36619b61ffa2429eeaefaab4f3374666adf36ad8ac6330d855848d7d36fd/ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b82a7c94a498853aa0b272fd5bc67f29008da798d4f93a2f9f289feb8426a58d", size = 715692, upload-time = "2024-12-11T19:58:17.252Z" }, - { url = "https://files.pythonhosted.org/packages/b1/82/85cb92f15a4231c89b95dfe08b09eb6adca929ef7df7e17ab59902b6f589/ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5", size = 98777, upload-time = "2024-10-20T10:13:01.395Z" }, - { url = "https://files.pythonhosted.org/packages/d7/8f/c3654f6f1ddb75daf3922c3d8fc6005b1ab56671ad56ffb874d908bfa668/ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4", size = 115523, upload-time = "2024-10-20T10:13:02.768Z" }, - { url = "https://files.pythonhosted.org/packages/29/00/4864119668d71a5fa45678f380b5923ff410701565821925c69780356ffa/ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a", size = 132011, upload-time = "2024-10-20T10:13:04.377Z" }, - { url = "https://files.pythonhosted.org/packages/7f/5e/212f473a93ae78c669ffa0cb051e3fee1139cb2d385d2ae1653d64281507/ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475", size = 642488, upload-time = "2024-10-20T10:13:05.906Z" }, - { url = "https://files.pythonhosted.org/packages/1f/8f/ecfbe2123ade605c49ef769788f79c38ddb1c8fa81e01f4dbf5cf1a44b16/ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef", size = 745066, upload-time = "2024-10-20T10:13:07.26Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a9/28f60726d29dfc01b8decdb385de4ced2ced9faeb37a847bd5cf26836815/ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6", size = 701785, upload-time = "2024-10-20T10:13:08.504Z" }, - { url = "https://files.pythonhosted.org/packages/84/7e/8e7ec45920daa7f76046578e4f677a3215fe8f18ee30a9cb7627a19d9b4c/ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf", size = 693017, upload-time = "2024-10-21T11:26:48.866Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b3/d650eaade4ca225f02a648321e1ab835b9d361c60d51150bac49063b83fa/ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1", size = 741270, upload-time = "2024-10-21T11:26:50.213Z" }, - { url = "https://files.pythonhosted.org/packages/87/b8/01c29b924dcbbed75cc45b30c30d565d763b9c4d540545a0eeecffb8f09c/ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f6f3eac23941b32afccc23081e1f50612bdbe4e982012ef4f5797986828cd01", size = 709059, upload-time = "2024-12-11T19:58:18.846Z" }, - { url = "https://files.pythonhosted.org/packages/30/8c/ed73f047a73638257aa9377ad356bea4d96125b305c34a28766f4445cc0f/ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6", size = 98583, upload-time = "2024-10-20T10:13:09.658Z" }, - { url = "https://files.pythonhosted.org/packages/b0/85/e8e751d8791564dd333d5d9a4eab0a7a115f7e349595417fd50ecae3395c/ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3", size = 115190, upload-time = "2024-10-20T10:13:10.66Z" }, -] - -[[package]] -name = "semver" -version = "3.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, -] - -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, + { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] [[package]] @@ -2245,48 +1789,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.43" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "(python_full_version < '3.14' and platform_machine == 'AMD64') or (python_full_version < '3.14' and platform_machine == 'WIN32') or (python_full_version < '3.14' and platform_machine == 'aarch64') or (python_full_version < '3.14' and platform_machine == 'amd64') or (python_full_version < '3.14' and platform_machine == 'ppc64le') or (python_full_version < '3.14' and platform_machine == 'win32') or (python_full_version < '3.14' and platform_machine == 'x86_64')" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/bc/d59b5d97d27229b0e009bd9098cd81af71c2fa5549c580a0a67b9bed0496/sqlalchemy-2.0.43.tar.gz", hash = "sha256:788bfcef6787a7764169cfe9859fe425bf44559619e1d9f56f5bddf2ebf6f417", size = 9762949, upload-time = "2025-08-11T14:24:58.438Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/77/fa7189fe44114658002566c6fe443d3ed0ec1fa782feb72af6ef7fbe98e7/sqlalchemy-2.0.43-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:52d9b73b8fb3e9da34c2b31e6d99d60f5f99fd8c1225c9dad24aeb74a91e1d29", size = 2136472, upload-time = "2025-08-11T15:52:21.789Z" }, - { url = "https://files.pythonhosted.org/packages/99/ea/92ac27f2fbc2e6c1766bb807084ca455265707e041ba027c09c17d697867/sqlalchemy-2.0.43-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f42f23e152e4545157fa367b2435a1ace7571cab016ca26038867eb7df2c3631", size = 2126535, upload-time = "2025-08-11T15:52:23.109Z" }, - { url = "https://files.pythonhosted.org/packages/94/12/536ede80163e295dc57fff69724caf68f91bb40578b6ac6583a293534849/sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fb1a8c5438e0c5ea51afe9c6564f951525795cf432bed0c028c1cb081276685", size = 3297521, upload-time = "2025-08-11T15:50:33.536Z" }, - { url = "https://files.pythonhosted.org/packages/03/b5/cacf432e6f1fc9d156eca0560ac61d4355d2181e751ba8c0cd9cb232c8c1/sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db691fa174e8f7036afefe3061bc40ac2b770718be2862bfb03aabae09051aca", size = 3297343, upload-time = "2025-08-11T15:57:51.186Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ba/d4c9b526f18457667de4c024ffbc3a0920c34237b9e9dd298e44c7c00ee5/sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2b3b4927d0bc03d02ad883f402d5de201dbc8894ac87d2e981e7d87430e60d", size = 3232113, upload-time = "2025-08-11T15:50:34.949Z" }, - { url = "https://files.pythonhosted.org/packages/aa/79/c0121b12b1b114e2c8a10ea297a8a6d5367bc59081b2be896815154b1163/sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d3d9b904ad4a6b175a2de0738248822f5ac410f52c2fd389ada0b5262d6a1e3", size = 3258240, upload-time = "2025-08-11T15:57:52.983Z" }, - { url = "https://files.pythonhosted.org/packages/79/99/a2f9be96fb382f3ba027ad42f00dbe30fdb6ba28cda5f11412eee346bec5/sqlalchemy-2.0.43-cp311-cp311-win32.whl", hash = "sha256:5cda6b51faff2639296e276591808c1726c4a77929cfaa0f514f30a5f6156921", size = 2101248, upload-time = "2025-08-11T15:55:01.855Z" }, - { url = "https://files.pythonhosted.org/packages/ee/13/744a32ebe3b4a7a9c7ea4e57babae7aa22070d47acf330d8e5a1359607f1/sqlalchemy-2.0.43-cp311-cp311-win_amd64.whl", hash = "sha256:c5d1730b25d9a07727d20ad74bc1039bbbb0a6ca24e6769861c1aa5bf2c4c4a8", size = 2126109, upload-time = "2025-08-11T15:55:04.092Z" }, - { url = "https://files.pythonhosted.org/packages/61/db/20c78f1081446095450bdc6ee6cc10045fce67a8e003a5876b6eaafc5cc4/sqlalchemy-2.0.43-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:20d81fc2736509d7a2bd33292e489b056cbae543661bb7de7ce9f1c0cd6e7f24", size = 2134891, upload-time = "2025-08-11T15:51:13.019Z" }, - { url = "https://files.pythonhosted.org/packages/45/0a/3d89034ae62b200b4396f0f95319f7d86e9945ee64d2343dcad857150fa2/sqlalchemy-2.0.43-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b9fc27650ff5a2c9d490c13c14906b918b0de1f8fcbb4c992712d8caf40e83", size = 2123061, upload-time = "2025-08-11T15:51:14.319Z" }, - { url = "https://files.pythonhosted.org/packages/cb/10/2711f7ff1805919221ad5bee205971254845c069ee2e7036847103ca1e4c/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6772e3ca8a43a65a37c88e2f3e2adfd511b0b1da37ef11ed78dea16aeae85bd9", size = 3320384, upload-time = "2025-08-11T15:52:35.088Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0e/3d155e264d2ed2778484006ef04647bc63f55b3e2d12e6a4f787747b5900/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a113da919c25f7f641ffbd07fbc9077abd4b3b75097c888ab818f962707eb48", size = 3329648, upload-time = "2025-08-11T15:56:34.153Z" }, - { url = "https://files.pythonhosted.org/packages/5b/81/635100fb19725c931622c673900da5efb1595c96ff5b441e07e3dd61f2be/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4286a1139f14b7d70141c67a8ae1582fc2b69105f1b09d9573494eb4bb4b2687", size = 3258030, upload-time = "2025-08-11T15:52:36.933Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ed/a99302716d62b4965fded12520c1cbb189f99b17a6d8cf77611d21442e47/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:529064085be2f4d8a6e5fab12d36ad44f1909a18848fcfbdb59cc6d4bbe48efe", size = 3294469, upload-time = "2025-08-11T15:56:35.553Z" }, - { url = "https://files.pythonhosted.org/packages/5d/a2/3a11b06715149bf3310b55a98b5c1e84a42cfb949a7b800bc75cb4e33abc/sqlalchemy-2.0.43-cp312-cp312-win32.whl", hash = "sha256:b535d35dea8bbb8195e7e2b40059e2253acb2b7579b73c1b432a35363694641d", size = 2098906, upload-time = "2025-08-11T15:55:00.645Z" }, - { url = "https://files.pythonhosted.org/packages/bc/09/405c915a974814b90aa591280623adc6ad6b322f61fd5cff80aeaef216c9/sqlalchemy-2.0.43-cp312-cp312-win_amd64.whl", hash = "sha256:1c6d85327ca688dbae7e2b06d7d84cfe4f3fffa5b5f9e21bb6ce9d0e1a0e0e0a", size = 2126260, upload-time = "2025-08-11T15:55:02.965Z" }, - { url = "https://files.pythonhosted.org/packages/41/1c/a7260bd47a6fae7e03768bf66451437b36451143f36b285522b865987ced/sqlalchemy-2.0.43-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e7c08f57f75a2bb62d7ee80a89686a5e5669f199235c6d1dac75cd59374091c3", size = 2130598, upload-time = "2025-08-11T15:51:15.903Z" }, - { url = "https://files.pythonhosted.org/packages/8e/84/8a337454e82388283830b3586ad7847aa9c76fdd4f1df09cdd1f94591873/sqlalchemy-2.0.43-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14111d22c29efad445cd5021a70a8b42f7d9152d8ba7f73304c4d82460946aaa", size = 2118415, upload-time = "2025-08-11T15:51:17.256Z" }, - { url = "https://files.pythonhosted.org/packages/cf/ff/22ab2328148492c4d71899d62a0e65370ea66c877aea017a244a35733685/sqlalchemy-2.0.43-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21b27b56eb2f82653168cefe6cb8e970cdaf4f3a6cb2c5e3c3c1cf3158968ff9", size = 3248707, upload-time = "2025-08-11T15:52:38.444Z" }, - { url = "https://files.pythonhosted.org/packages/dc/29/11ae2c2b981de60187f7cbc84277d9d21f101093d1b2e945c63774477aba/sqlalchemy-2.0.43-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c5a9da957c56e43d72126a3f5845603da00e0293720b03bde0aacffcf2dc04f", size = 3253602, upload-time = "2025-08-11T15:56:37.348Z" }, - { url = "https://files.pythonhosted.org/packages/b8/61/987b6c23b12c56d2be451bc70900f67dd7d989d52b1ee64f239cf19aec69/sqlalchemy-2.0.43-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d79f9fdc9584ec83d1b3c75e9f4595c49017f5594fee1a2217117647225d738", size = 3183248, upload-time = "2025-08-11T15:52:39.865Z" }, - { url = "https://files.pythonhosted.org/packages/86/85/29d216002d4593c2ce1c0ec2cec46dda77bfbcd221e24caa6e85eff53d89/sqlalchemy-2.0.43-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9df7126fd9db49e3a5a3999442cc67e9ee8971f3cb9644250107d7296cb2a164", size = 3219363, upload-time = "2025-08-11T15:56:39.11Z" }, - { url = "https://files.pythonhosted.org/packages/b6/e4/bd78b01919c524f190b4905d47e7630bf4130b9f48fd971ae1c6225b6f6a/sqlalchemy-2.0.43-cp313-cp313-win32.whl", hash = "sha256:7f1ac7828857fcedb0361b48b9ac4821469f7694089d15550bbcf9ab22564a1d", size = 2096718, upload-time = "2025-08-11T15:55:05.349Z" }, - { url = "https://files.pythonhosted.org/packages/ac/a5/ca2f07a2a201f9497de1928f787926613db6307992fe5cda97624eb07c2f/sqlalchemy-2.0.43-cp313-cp313-win_amd64.whl", hash = "sha256:971ba928fcde01869361f504fcff3b7143b47d30de188b11c6357c0505824197", size = 2123200, upload-time = "2025-08-11T15:55:07.932Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d9/13bdde6521f322861fab67473cec4b1cc8999f3871953531cf61945fad92/sqlalchemy-2.0.43-py3-none-any.whl", hash = "sha256:1681c21dd2ccee222c2fe0bef671d1aef7c504087c9c4e800371cfcc8ac966fc", size = 1924759, upload-time = "2025-08-11T15:39:53.024Z" }, -] - -[package.optional-dependencies] -asyncio = [ - { name = "greenlet" }, -] - [[package]] name = "sse-starlette" version = "3.0.2" @@ -2313,36 +1815,70 @@ wheels = [ ] [[package]] -name = "text-unidecode" -version = "1.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, -] - -[[package]] -name = "toml" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, -] - -[[package]] -name = "typer" -version = "0.17.4" +name = "temporalio" +version = "1.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, + { name = "nexus-rpc" }, + { name = "protobuf" }, + { name = "types-protobuf" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/e8/2a73ccf9874ec4c7638f172efc8972ceab13a0e3480b389d6ed822f7a822/typer-0.17.4.tar.gz", hash = "sha256:b77dc07d849312fd2bb5e7f20a7af8985c7ec360c45b051ed5412f64d8dc1580", size = 103734, upload-time = "2025-09-05T18:14:40.746Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/7a/9f7885950cc040d71340a9379134b168d557b0a0e589c75d31e797f5a8bf/temporalio-1.18.1.tar.gz", hash = "sha256:46394498f8822e61b3ce70d6735de7618f5af0501fb90f3f90f4b4f9e7816d77", size = 1787082, upload-time = "2025-09-30T15:00:19.871Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/72/6b3e70d32e89a5cbb6a4513726c1ae8762165b027af569289e19ec08edd8/typer-0.17.4-py3-none-any.whl", hash = "sha256:015534a6edaa450e7007eba705d5c18c3349dcea50a6ad79a5ed530967575824", size = 46643, upload-time = "2025-09-05T18:14:39.166Z" }, + { url = "https://files.pythonhosted.org/packages/82/c0/9bad907dcf968c55acee1b5cc4ec0590a0fca3bc448dc32898785a577f7b/temporalio-1.18.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:748c0ec9f48aa1ab612a58fe516d9be28c1dd98194f560fd28a2ab09c6e2ca5e", size = 12809719, upload-time = "2025-09-30T14:59:58.177Z" }, + { url = "https://files.pythonhosted.org/packages/51/c5/490a2726aa67d4b856e8288d36848e7859801889b21d251cae8e8a6c9311/temporalio-1.18.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5a789e7c483582d6d7dd49e7d2d2730d82dc94d9342fe71be76fa67afa4e6865", size = 12393639, upload-time = "2025-09-30T15:00:02.737Z" }, + { url = "https://files.pythonhosted.org/packages/92/89/e500e066df3c0fc1e6ee1a7cadbdfbc9812c62296ac0554fc09779555560/temporalio-1.18.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9f5cf75c4b887476a2b39d022a9c44c495f5eb1668087a022bd9258d3adddf9", size = 12732719, upload-time = "2025-09-30T15:00:07.458Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/7e5c4082b1550c38c802af02ae60ffe39d87646856aa51909cdd2789b7a6/temporalio-1.18.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f28a69394bf18b4a1c22a6a784d348e93482858c505d054570b278f0f5e13e9c", size = 12926861, upload-time = "2025-09-30T15:00:12.777Z" }, + { url = "https://files.pythonhosted.org/packages/10/49/e021b3205f06a1ec8a533dc8b02dcf5784d003cf99e4fd574eedb7439357/temporalio-1.18.1-cp39-abi3-win_amd64.whl", hash = "sha256:552b360f9ccdac8d5fc5d19c6578c2f6f634399ccc37439c4794aa58487f7fd5", size = 13059005, upload-time = "2025-09-30T15:00:17.586Z" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "types-protobuf" +version = "6.32.1.20250918" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/5a/bd06c2dbb77ebd4ea764473c9c4c014c7ba94432192cb965a274f8544b9d/types_protobuf-6.32.1.20250918.tar.gz", hash = "sha256:44ce0ae98475909ca72379946ab61a4435eec2a41090821e713c17e8faf5b88f", size = 63780, upload-time = "2025-09-18T02:50:39.391Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/5a/8d93d4f4af5dc3dd62aa4f020deae746b34b1d94fb5bee1f776c6b7e9d6c/types_protobuf-6.32.1.20250918-py3-none-any.whl", hash = "sha256:22ba6133d142d11cc34d3788ad6dead2732368ebb0406eaa7790ea6ae46c8d0b", size = 77885, upload-time = "2025-09-18T02:50:38.028Z" }, ] [[package]] @@ -2366,27 +1902,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, ] -[[package]] -name = "tzdata" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, -] - -[[package]] -name = "tzlocal" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, -] - [[package]] name = "urllib3" version = "2.5.0" @@ -2396,32 +1911,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, ] -[[package]] -name = "uv" -version = "0.8.17" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/4c/c270c6b8ed3e8c7fe38ea0b99df9eff09c332421b93d55a158371f75220e/uv-0.8.17.tar.gz", hash = "sha256:2afd4525a53c8ab3a11a5a15093c503d27da67e76257a649b05e4f0bc2ebb5ae", size = 3615060, upload-time = "2025-09-10T21:51:25.067Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/7d/bbaa45c88b2c91e02714a8a5c9e787c47e4898bddfdd268569163492ba45/uv-0.8.17-py3-none-linux_armv6l.whl", hash = "sha256:c51c9633ca93ef63c07df2443941e6264efd2819cc9faabfd9fe11899c6a0d6a", size = 20242144, upload-time = "2025-09-10T21:50:18.081Z" }, - { url = "https://files.pythonhosted.org/packages/65/34/609b72034df0c62bcfb0c0ad4b11e2b55e537c0f0817588b5337d3dcca71/uv-0.8.17-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c28fba6d7bb5c34ade2c8da5000faebe8425a287f42a043ca01ceb24ebc81590", size = 19363081, upload-time = "2025-09-10T21:50:22.731Z" }, - { url = "https://files.pythonhosted.org/packages/b6/bc/9417df48f0c18a9d54c2444096e03f2f56a3534c5b869f50ac620729cbc8/uv-0.8.17-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b009f1ec9e28de00f76814ad66e35aaae82c98a0f24015de51943dcd1c2a1895", size = 17943513, upload-time = "2025-09-10T21:50:25.824Z" }, - { url = "https://files.pythonhosted.org/packages/63/1c/14fd54c852fd592a2b5da4b7960f3bf4a15c7e51eb20eaddabe8c8cca32d/uv-0.8.17-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:84d56ae50ca71aec032577adf9737974554a82a94e52cee57722745656c1d383", size = 19507222, upload-time = "2025-09-10T21:50:29.237Z" }, - { url = "https://files.pythonhosted.org/packages/be/47/f6a68cc310feca37c965bcbd57eb999e023d35eaeda9c9759867bf3ed232/uv-0.8.17-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:85c2140f8553b9a4387a7395dc30cd151ef94046785fe8b198f13f2c380fb39b", size = 19865652, upload-time = "2025-09-10T21:50:32.87Z" }, - { url = "https://files.pythonhosted.org/packages/ab/6a/fdeb2d4a2635a6927c6d549b07177bcaf6ce15bdef58e8253e75c1b70f54/uv-0.8.17-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2076119783e4a6d3c9e25638956cb123f0eabf4d7d407d9661cdf7f84818dcb9", size = 20831760, upload-time = "2025-09-10T21:50:37.803Z" }, - { url = "https://files.pythonhosted.org/packages/d0/4c/bd58b8a76015aa9ac49d6b4e1211ae1ca98a0aade0c49e1a5f645fb5cd38/uv-0.8.17-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:707a55660d302924fdbcb509e63dfec8842e19d35b69bcc17af76c25db15ad6f", size = 22209056, upload-time = "2025-09-10T21:50:41.749Z" }, - { url = "https://files.pythonhosted.org/packages/7e/2e/28f59c00a2ed6532502fb1e27da9394e505fb7b41cc0274475104b43561b/uv-0.8.17-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1824b76911a14aaa9eee65ad9e180e6a4d2d7c86826232c2f28ae86aee56ed0e", size = 21871684, upload-time = "2025-09-10T21:50:45.331Z" }, - { url = "https://files.pythonhosted.org/packages/5a/1d/a8a4fc08de1f767316467e7a1989bb125734b7ed9cd98ce8969386a70653/uv-0.8.17-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bb9b515cc813fb1b08f1e7592f76e437e2fb44945e53cde4fee11dee3b16d0c3", size = 21145154, upload-time = "2025-09-10T21:50:50.388Z" }, - { url = "https://files.pythonhosted.org/packages/8f/35/cb47d2d07a383c07b0e5043c6fe5555f0fd79683c6d7f9760222987c8be9/uv-0.8.17-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d30d02fb65193309fc12a20f9e1a9fab67f469d3e487a254ca1145fd06788f", size = 21106619, upload-time = "2025-09-10T21:50:54.5Z" }, - { url = "https://files.pythonhosted.org/packages/6e/93/c310f0153b9dfe79bdd7f7eaef6380a8545c8939dbfc4e6bdee8f3ee7050/uv-0.8.17-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:3941cecd9a6a46d3d4505753912c9cf3e8ae5eea30b9d0813f3656210f8c5d01", size = 19777591, upload-time = "2025-09-10T21:50:57.765Z" }, - { url = "https://files.pythonhosted.org/packages/6c/4f/971d3c84c2f09cf8df4536c33644e6b97e10a259d8630a0c1696c1fa6e94/uv-0.8.17-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:cd0ad366cfe4cbe9212bd660b5b9f3a827ff35a7601cefdac2d153bfc8079eb7", size = 20845039, upload-time = "2025-09-10T21:51:01.167Z" }, - { url = "https://files.pythonhosted.org/packages/4a/29/8ad9038e75cb91f54b81cc933dd14fcfa92fa6f8706117d43d4251a8a662/uv-0.8.17-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:505854bc75c497b95d2c65590291dc820999a4a7d9dfab4f44a9434a6cff7b5f", size = 19820370, upload-time = "2025-09-10T21:51:04.616Z" }, - { url = "https://files.pythonhosted.org/packages/f2/c9/fc8482d1e7dfe187c6e03dcefbac0db41a5dd72aa7b017c0f80f91a04444/uv-0.8.17-py3-none-musllinux_1_1_i686.whl", hash = "sha256:dc479f661da449df37d68b36fdffa641e89fb53ad38c16a5c9f98f3211785b63", size = 20289951, upload-time = "2025-09-10T21:51:08.605Z" }, - { url = "https://files.pythonhosted.org/packages/2d/84/ad878ed045f02aa973be46636c802d494f8270caf5ea8bd04b7bbc68aa23/uv-0.8.17-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:a1d11cd805be6d137ffef4a8227905f87f459031c645ac5031c30a3bcd08abd6", size = 21234644, upload-time = "2025-09-10T21:51:12.429Z" }, - { url = "https://files.pythonhosted.org/packages/f8/03/3fa2641513922988e641050b3adbc87de527f44c2cc8328510703616be6a/uv-0.8.17-py3-none-win32.whl", hash = "sha256:d13a616eb0b2b33c7aa09746cc85860101d595655b58653f0b499af19f33467c", size = 19216757, upload-time = "2025-09-10T21:51:16.021Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c4/0082f437bac162ab95e5a3a389a184c122d45eb5593960aab92fdf80374b/uv-0.8.17-py3-none-win_amd64.whl", hash = "sha256:cf85b84b81b41d57a9b6eeded8473ec06ace8ee959ad0bb57e102b5ad023bd34", size = 21125811, upload-time = "2025-09-10T21:51:19.397Z" }, - { url = "https://files.pythonhosted.org/packages/50/a2/29f57b118b3492c9d5ab1a99ba4906e7d7f8b658881d31bc2c4408d64d07/uv-0.8.17-py3-none-win_arm64.whl", hash = "sha256:64d649a8c4c3732b05dc712544963b004cf733d95fdc5d26f43c5493553ff0a7", size = 19564631, upload-time = "2025-09-10T21:51:22.599Z" }, -] - [[package]] name = "uvicorn" version = "0.35.0" @@ -2435,48 +1924,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/e2/dc81b1bd1dcfe91735810265e9d26bc8ec5da45b4c0f6237e286819194c3/uvicorn-0.35.0-py3-none-any.whl", hash = "sha256:197535216b25ff9b785e29a0b79199f55222193d47f820816e7da751e9bc8d4a", size = 66406, upload-time = "2025-06-28T16:15:44.816Z" }, ] -[[package]] -name = "websockets" -version = "15.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/32/18fcd5919c293a398db67443acd33fde142f283853076049824fc58e6f75/websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431", size = 175423, upload-time = "2025-03-05T20:01:56.276Z" }, - { url = "https://files.pythonhosted.org/packages/76/70/ba1ad96b07869275ef42e2ce21f07a5b0148936688c2baf7e4a1f60d5058/websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57", size = 173082, upload-time = "2025-03-05T20:01:57.563Z" }, - { url = "https://files.pythonhosted.org/packages/86/f2/10b55821dd40eb696ce4704a87d57774696f9451108cff0d2824c97e0f97/websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905", size = 173330, upload-time = "2025-03-05T20:01:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/a5/90/1c37ae8b8a113d3daf1065222b6af61cc44102da95388ac0018fcb7d93d9/websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562", size = 182878, upload-time = "2025-03-05T20:02:00.305Z" }, - { url = "https://files.pythonhosted.org/packages/8e/8d/96e8e288b2a41dffafb78e8904ea7367ee4f891dafc2ab8d87e2124cb3d3/websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792", size = 181883, upload-time = "2025-03-05T20:02:03.148Z" }, - { url = "https://files.pythonhosted.org/packages/93/1f/5d6dbf551766308f6f50f8baf8e9860be6182911e8106da7a7f73785f4c4/websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413", size = 182252, upload-time = "2025-03-05T20:02:05.29Z" }, - { url = "https://files.pythonhosted.org/packages/d4/78/2d4fed9123e6620cbf1706c0de8a1632e1a28e7774d94346d7de1bba2ca3/websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8", size = 182521, upload-time = "2025-03-05T20:02:07.458Z" }, - { url = "https://files.pythonhosted.org/packages/e7/3b/66d4c1b444dd1a9823c4a81f50231b921bab54eee2f69e70319b4e21f1ca/websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3", size = 181958, upload-time = "2025-03-05T20:02:09.842Z" }, - { url = "https://files.pythonhosted.org/packages/08/ff/e9eed2ee5fed6f76fdd6032ca5cd38c57ca9661430bb3d5fb2872dc8703c/websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf", size = 181918, upload-time = "2025-03-05T20:02:11.968Z" }, - { url = "https://files.pythonhosted.org/packages/d8/75/994634a49b7e12532be6a42103597b71098fd25900f7437d6055ed39930a/websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85", size = 176388, upload-time = "2025-03-05T20:02:13.32Z" }, - { url = "https://files.pythonhosted.org/packages/98/93/e36c73f78400a65f5e236cd376713c34182e6663f6889cd45a4a04d8f203/websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065", size = 176828, upload-time = "2025-03-05T20:02:14.585Z" }, - { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, - { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, - { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, - { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, - { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, - { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, - { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, - { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, - { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, - { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, - { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, - { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" }, - { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" }, - { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" }, - { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" }, - { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" }, - { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" }, - { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" }, - { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, - { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, -] - [[package]] name = "werkzeug" version = "3.1.1" @@ -2489,60 +1936,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/ea/c67e1dee1ba208ed22c06d1d547ae5e293374bfc43e0eb0ef5e262b68561/werkzeug-3.1.1-py3-none-any.whl", hash = "sha256:a71124d1ef06008baafa3d266c02f56e1836a5984afd6dd6c9230669d60d9fb5", size = 224371, upload-time = "2024-11-01T16:40:43.994Z" }, ] -[[package]] -name = "whenever" -version = "0.8.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "python_full_version >= '3.13' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4a/b6/0e871022a7a5ec9c80c3e19028c806a7a079b3e0aaa524e1a8ccfd52e6bf/whenever-0.8.8.tar.gz", hash = "sha256:d0674d410fbbcf495f6cca0f1f575279e402887d20e4c1ca7d11309cd41b8125", size = 235496, upload-time = "2025-07-24T20:59:42.467Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/3d/240aa3d1dc6627e633470449f2ac8e8179e8d5b4e6c41ca5e805c0776f68/whenever-0.8.8-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0cbdcba2df308f09d26db924459d31ee5c5bcb09e72a16bfb9fd7cdd05812920", size = 390101, upload-time = "2025-07-24T20:59:22.969Z" }, - { url = "https://files.pythonhosted.org/packages/87/f0/b4bcdd4d8edeec098d8acbdedb35c066559a51752e383c6a3f944d4e3703/whenever-0.8.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:943f1e4054afc664b79b44929569598513587978395ef159340253ed0bf73e6e", size = 374988, upload-time = "2025-07-24T20:59:15.677Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a6/be07090cedde0fd27be569012609b933eb22c49e651e59cbba96951691da/whenever-0.8.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc6fe69fe491751c91d9d5a11bd18e3e15fa6d0d294e661d71e9df7e8cee3f9e", size = 397112, upload-time = "2025-07-24T20:58:15.481Z" }, - { url = "https://files.pythonhosted.org/packages/48/08/1d9c49c8f35afc6342eded722d11503f6b0105b66e4861d6776a05111960/whenever-0.8.8-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1f6b813e15f845f9f8be7d6226cafc5cbd89746289fb302ad40213f28d1911ec", size = 436670, upload-time = "2025-07-24T20:58:27.947Z" }, - { url = "https://files.pythonhosted.org/packages/d5/78/6c616a30a518afecd7135928b6ca741ab274ab2021b631a21724cd056ae9/whenever-0.8.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aea29c76221f5d0ccdbbaf806c3b181c1aba9efade5726d501ce26ebed70b692", size = 430732, upload-time = "2025-07-24T20:58:39.876Z" }, - { url = "https://files.pythonhosted.org/packages/6c/aa/1ea65baee041b268ddd8352e3582bd6c3fb87c1f77329b44ac61ab59e22c/whenever-0.8.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5f331f03bdce04d9709053d49dd982c54e54f21005580273f3564327b42fb242", size = 450804, upload-time = "2025-07-24T20:58:46.399Z" }, - { url = "https://files.pythonhosted.org/packages/3b/fb/cddfc11b0c36787b746052e3e4741d2b61bcdf5636582d9f57ad5ca0b0cc/whenever-0.8.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5f98dc66db6016660897e0fa56904e75de35daeacac5a012bf4424ef4d567c5", size = 412014, upload-time = "2025-07-24T20:59:03.578Z" }, - { url = "https://files.pythonhosted.org/packages/ff/57/3d79a393c66e3999408f746159ba977b3ef5494b95503b3a1ac687251969/whenever-0.8.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:182d4ed4df2f09f173e09c606a866bc324f2ae4a67145f391cb669e915c4b0f4", size = 450866, upload-time = "2025-07-24T20:58:51.841Z" }, - { url = "https://files.pythonhosted.org/packages/c8/a8/8cff5c1eb4771b999f96e5a784a30db52433f69be56193cf146059eb4174/whenever-0.8.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56d3227e3212375f4e3de4c97a006de5e7973f1273b7aedcf69f7c883762ccc3", size = 574881, upload-time = "2025-07-24T20:58:22.024Z" }, - { url = "https://files.pythonhosted.org/packages/a5/04/f423acad21fa11cf0240b417e04d9af3b74eb58e15820e500229874c69e5/whenever-0.8.8-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c21a8ef5ad6d03d473d782444b2b99a0e65d48ab11951ce11858896aebaf459b", size = 699936, upload-time = "2025-07-24T20:58:34.252Z" }, - { url = "https://files.pythonhosted.org/packages/dd/d5/0c2c8be9e2376417203d950ca77c2df2132018863cde7c73154a114e72b2/whenever-0.8.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:325ae450a6f476837497a4391e7ddc7353f3f944061a5f607f48b9fc3f5d98c0", size = 625557, upload-time = "2025-07-24T20:58:57.795Z" }, - { url = "https://files.pythonhosted.org/packages/29/ce/d17170b7487da4b2c95960499d14838197645cb879c0862dcb35a4edf953/whenever-0.8.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9f4118fd28090c914e8be577ffded3508e53adbbff39c17df03611d16c68c0b2", size = 583281, upload-time = "2025-07-24T20:59:09.875Z" }, - { url = "https://files.pythonhosted.org/packages/50/b8/83ba57d8e14c7769f00096aa8de4d8022a034e402ce780abaf56a678e9d4/whenever-0.8.8-cp311-cp311-win32.whl", hash = "sha256:cf4147a361595da9fa981f35e95f5acd58316137ad97d140630407794b280c75", size = 328290, upload-time = "2025-07-24T20:59:28.821Z" }, - { url = "https://files.pythonhosted.org/packages/af/74/44253162e1b25fe2b70bde217ef0cd76f7811484a9bbed00df769fabffb6/whenever-0.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:9faeee933c9fd22df352c55e5a19ae3959187971ee3d2aecfdded2fd6f4a86e8", size = 320868, upload-time = "2025-07-24T20:59:35.631Z" }, - { url = "https://files.pythonhosted.org/packages/9c/67/562a89b70ed28bef984a45c022c320ceea6722054b8bb2d54ad089c87978/whenever-0.8.8-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:43a0a56b2b2bb6f821161fa4e0c077e24909d02241132f8aad47a5ad604f4239", size = 391248, upload-time = "2025-07-24T20:59:24.12Z" }, - { url = "https://files.pythonhosted.org/packages/e6/2d/d10a544d536b6fcf807f06355cfc36298c14b584999947932afe16083f53/whenever-0.8.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82988062e7f8695d1d567e9d259ebeb90f0d43c1b1e34d12019019887375709e", size = 375095, upload-time = "2025-07-24T20:59:16.852Z" }, - { url = "https://files.pythonhosted.org/packages/2d/16/1844839a6db4de85d8da06beabab2a1207161f40d60bb13a0a5a89b2f997/whenever-0.8.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e929a3a2815bc5e2dd53504afec714a837c99b4b67dbb261e8594b20f395eaf", size = 396763, upload-time = "2025-07-24T20:58:16.971Z" }, - { url = "https://files.pythonhosted.org/packages/2d/fc/7a4789b8f9c6335a9ecb027034a65caabe256a140389d3d04ee711070179/whenever-0.8.8-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c97861c051babbcf15b4e6ba021d52941096d4a0c46bf98db162f3720f02725", size = 437276, upload-time = "2025-07-24T20:58:29.35Z" }, - { url = "https://files.pythonhosted.org/packages/70/a3/ba827598a30ae11b4e9a5109a295f66cb3d90c7d19456bc9c886d8f84a75/whenever-0.8.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5731f7d0dc7226f88cc40facc7dfe3810fa921930e19b99cbf4086472e411b6", size = 432758, upload-time = "2025-07-24T20:58:41.132Z" }, - { url = "https://files.pythonhosted.org/packages/37/db/49a94b6ace1a3c3d8633fc45b7c87c604005564e5848bcafc548b321ccfb/whenever-0.8.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69f745b81e9f75b5fdd39596d39dfa05f8a9d7288de5b3782bb30d590a310e12", size = 451512, upload-time = "2025-07-24T20:58:47.429Z" }, - { url = "https://files.pythonhosted.org/packages/b3/f4/7c4b828a7e2d8efe16ec989c11522bce8b5c8420a931a7acb97b5028f33c/whenever-0.8.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52e19d4484413f5c08e5e1f5b8968efc119ce7a8bfe788aab136c5c93a33b93b", size = 412824, upload-time = "2025-07-24T20:59:04.697Z" }, - { url = "https://files.pythonhosted.org/packages/33/c0/0cebf03683013a0498c48b787be1b8af9b779cc35230f3645cf3c7497c5d/whenever-0.8.8-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:068232acf18432d86e75286b4b5bc6e3d9b4542a8f9b9ba0114e2aa9cdb4778c", size = 452049, upload-time = "2025-07-24T20:58:52.919Z" }, - { url = "https://files.pythonhosted.org/packages/02/fe/16608f4f30c1262a8d9e343e255908772172734331db19d113b6ff5fd0f3/whenever-0.8.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:65889c31520e5241a619b17a82f30252a1b9e9f3dbaf813b1de2b45f218a2c87", size = 574646, upload-time = "2025-07-24T20:58:23.152Z" }, - { url = "https://files.pythonhosted.org/packages/02/a5/40904c9f9583b2c8a666c3c90ff58c92722b5fd62428470bc0824036d480/whenever-0.8.8-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:07dad7e78e3f73516630c3d74636dc966c19ae8b5099cbbf9fdb8b52678385fe", size = 700550, upload-time = "2025-07-24T20:58:35.355Z" }, - { url = "https://files.pythonhosted.org/packages/40/ee/041dde993beaba3902ccfbc858a41ddf2782bb274a93b41b635e073aee66/whenever-0.8.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:800832cf213fe48d58a2b0273304831a7bbfe7341712c2fdc631d9cba92231b0", size = 627101, upload-time = "2025-07-24T20:58:59.06Z" }, - { url = "https://files.pythonhosted.org/packages/2f/2a/d85257e332953a77b2cfd27cd698cb1d226f165b3643e588fded9dd801da/whenever-0.8.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5267dd1977286c7e6231917112fb860b111e7c7c380f94e3725e0c13dd13fdd9", size = 584180, upload-time = "2025-07-24T20:59:11.059Z" }, - { url = "https://files.pythonhosted.org/packages/f5/0e/bf591dc9334ba3f44a88996a60a5650408901e45877ca47808976624657a/whenever-0.8.8-cp312-cp312-win32.whl", hash = "sha256:8766b96c97570c5138100a20488985db0c7f49ad078644f8982a0e3d64080dd3", size = 329897, upload-time = "2025-07-24T20:59:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/5b/cd/6e41a11ad795947dc747208a7199aa6be9f3b8b07afba0b153ea6d715a35/whenever-0.8.8-cp312-cp312-win_amd64.whl", hash = "sha256:cf24b466c6043fe4f0344cdbf6c09b166e4fea03f5d57b6a439eb26f391b6839", size = 322283, upload-time = "2025-07-24T20:59:36.766Z" }, - { url = "https://files.pythonhosted.org/packages/1f/31/5bf53c6ae051cec2ae23477bee58ea7e6d61ef1b4a5f012e8fd65fa14eec/whenever-0.8.8-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:809d56d01440b37b3dae4e3856ebf89322d51333bcddfecacacc235fff3f45c1", size = 391249, upload-time = "2025-07-24T20:59:25.297Z" }, - { url = "https://files.pythonhosted.org/packages/6a/95/0837ab424edb8568afecad9cf742bac1b4cd588bab8152ef26249c795c90/whenever-0.8.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a3b0e34a7afe367f245bcf3b8a9e581b6c09437d623001acf23207b08709837e", size = 375098, upload-time = "2025-07-24T20:59:18.543Z" }, - { url = "https://files.pythonhosted.org/packages/81/62/e28b36a7f478729cee8571bf1667ba77091f2d8638230af705cdd34e8450/whenever-0.8.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d1b517eab4cd6edd13cbd7236b5bb3d1babf0606dd756141a6cc274580cee56", size = 396770, upload-time = "2025-07-24T20:58:18.064Z" }, - { url = "https://files.pythonhosted.org/packages/77/e4/8706da917832a43999142afd7f2f68aabf6bd109fd69142237e61461f787/whenever-0.8.8-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24c0e9074b6f73d60eadd0d8149c0d29c5b98b054f881d0a1140367c43c9ec94", size = 437294, upload-time = "2025-07-24T20:58:30.399Z" }, - { url = "https://files.pythonhosted.org/packages/07/87/6d39cb9adf6ef982f3b8471f786fa1330bf73b021e2619204b8934b716b1/whenever-0.8.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b03e95f34d942e1e8c1c752d64e7166e7454112fbf6b4139d0eb9c017a17c6", size = 432762, upload-time = "2025-07-24T20:58:42.188Z" }, - { url = "https://files.pythonhosted.org/packages/c5/71/5f005ca40eb0d87804a3863f0c737cbd336e8d8bd234772809eb4dfa0ede/whenever-0.8.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c63453e0603bc4583661fbabe9b68ef41059b02f3bef9d572a9668e3dc74793", size = 451525, upload-time = "2025-07-24T20:58:48.577Z" }, - { url = "https://files.pythonhosted.org/packages/c7/61/b5e123dd98e90caeedd8ed3ede06b52ca0e18d8d16dfad8133c954702451/whenever-0.8.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1abba00ed463689d2d062eec36e5ea5f0d195d3fe6702121744592d5a234b09", size = 412819, upload-time = "2025-07-24T20:59:06.362Z" }, - { url = "https://files.pythonhosted.org/packages/93/a8/4897328754d8b7ffca335a473f5215380ad23651d19629e0c44290863ba3/whenever-0.8.8-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a05df53c1fe75df0a58c188971d63bf0ada627dcb74f2e0c2ff0fb3dd8d7d097", size = 452053, upload-time = "2025-07-24T20:58:54.04Z" }, - { url = "https://files.pythonhosted.org/packages/74/c1/fcd83e53e33d5b36e6b97f127a230dc9439cd5b3dcc5a6e2f546364b4c98/whenever-0.8.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fc2dde25b43a9375f7940504f8688259540d2d9960e5c973771d7b96030e3c95", size = 574642, upload-time = "2025-07-24T20:58:24.295Z" }, - { url = "https://files.pythonhosted.org/packages/d5/bc/22df727937940c0c4a3f0bf492c2d1ca5c03ab79307f0966a585e6159d34/whenever-0.8.8-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7c190f049532157a737a565a4797099684511011e2e1bd57e2488b0f5802a9e6", size = 700563, upload-time = "2025-07-24T20:58:36.411Z" }, - { url = "https://files.pythonhosted.org/packages/35/b8/143cabfd63ec0d05dbc9bfb6ba30f8d871ff19f5f34654602fb9ecbd572a/whenever-0.8.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:9815709758e58ceb751cfeb8ad222275e23b6ad512b5b7a956566bda70a1c6cd", size = 627103, upload-time = "2025-07-24T20:59:00.204Z" }, - { url = "https://files.pythonhosted.org/packages/2e/fd/f7359293eb3663f64da310db5a5e81be43943ffcdc27a3d62a92439f206f/whenever-0.8.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:605f0f06b6afbc8c86ca6c1c49a2b69cacb63d6af0fd683bdb1293a2635c3ce5", size = 584172, upload-time = "2025-07-24T20:59:12.186Z" }, - { url = "https://files.pythonhosted.org/packages/06/9b/6aaa8d1de1988e93e72135a7c84e02fd831833134383f2eca293def2562d/whenever-0.8.8-cp313-cp313-win32.whl", hash = "sha256:eb8fee01a0955aa9cbf4cc5b348a8562d7f1d277d9d6d0b9da601c0d45ad7f83", size = 329904, upload-time = "2025-07-24T20:59:31.173Z" }, - { url = "https://files.pythonhosted.org/packages/03/8f/9446a951cb423d76a046b90ba18403488d3282091453407e663af6bf4b2a/whenever-0.8.8-cp313-cp313-win_amd64.whl", hash = "sha256:fd71c8ac5cc761efe3b2dece32c5d22d44368155e37d948132b475bd804914d3", size = 322288, upload-time = "2025-07-24T20:59:37.901Z" }, - { url = "https://files.pythonhosted.org/packages/e8/1b/6fbe6e7d4a477f7ca2fe9d4f4fcce0e243a145b45ee35adc55dc577bffa7/whenever-0.8.8-py3-none-any.whl", hash = "sha256:b63d58613af9e44bed80d4a61ba0427db069bdede28ad5365b40bbe375a12990", size = 53489, upload-time = "2025-07-24T20:59:41.163Z" }, -] - [[package]] name = "yarl" version = "1.20.1" @@ -2624,12 +2017,3 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" }, { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, ] - -[[package]] -name = "zipp" -version = "3.23.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, -] diff --git a/cli/README.md b/cli/README.md index 5cd2e3c..47d0836 100644 --- a/cli/README.md +++ b/cli/README.md @@ -151,10 +151,10 @@ fuzzforge workflows parameters security_assessment --no-interactive ### Workflow Execution #### `fuzzforge workflow ` -Execute a security testing workflow. +Execute a security testing workflow with **automatic file upload**. ```bash -# Basic execution +# Basic execution - CLI automatically detects local files and uploads them fuzzforge workflow security_assessment /path/to/code # With parameters @@ -170,6 +170,49 @@ fuzzforge workflow security_assessment /path/to/code \ fuzzforge workflow security_assessment /path/to/code --wait ``` +**Automatic File Upload Behavior:** + +The CLI intelligently handles target files based on whether they exist locally: + +1. **Local file/directory exists** ā†’ **Automatic upload to MinIO**: + - CLI creates a compressed tarball (`.tar.gz`) for directories + - Uploads via HTTP to backend API + - Backend stores in MinIO with unique `target_id` + - Worker downloads from MinIO when ready to analyze + - āœ… **Works from any machine** (no shared filesystem needed) + +2. **Path doesn't exist locally** ā†’ **Path-based submission** (legacy): + - Path is sent to backend as-is + - Backend expects target to be accessible on its filesystem + - āš ļø Only works when CLI and backend share filesystem + +**Example workflow:** +```bash +$ ff workflow security_assessment ./my-project + +šŸ”§ Getting workflow information for: security_assessment +šŸ“¦ Detected local directory: ./my-project (21 files) +šŸ—œļø Creating compressed tarball... +šŸ“¤ Uploading to backend (0.01 MB)... +āœ… Upload complete! Target ID: 548193a1-f73f-4ec1-8068-19ec2660b8e4 + +šŸŽÆ Executing workflow: + Workflow: security_assessment + Target: my-project.tar.gz (uploaded) + Volume Mode: ro + Status: šŸ”„ RUNNING + +āœ… Workflow started successfully! + Execution ID: security_assessment-52781925 +``` + +**Upload Details:** +- **Max file size**: 10 GB (configurable on backend) +- **Compression**: Automatic for directories (reduces upload time) +- **Storage**: Files stored in MinIO (S3-compatible) +- **Lifecycle**: Automatic cleanup after 7 days +- **Caching**: Workers cache downloaded targets for faster repeated workflows + **Options:** - `--param, -p` - Parameter in key=value format (can be used multiple times) - `--param-file, -f` - JSON file containing parameters @@ -178,6 +221,22 @@ fuzzforge workflow security_assessment /path/to/code --wait - `--interactive/--no-interactive, -i/-n` - Interactive parameter input - `--wait, -w` - Wait for execution to complete +**Worker Lifecycle Options (v0.7.0):** +- `--auto-start/--no-auto-start` - Auto-start required worker (default: from config) +- `--auto-stop/--no-auto-stop` - Auto-stop worker after completion (default: from config) + +**Examples:** +```bash +# Worker starts automatically (default behavior) +fuzzforge workflow ossfuzz_campaign . project_name=zlib + +# Disable auto-start (worker must be running already) +fuzzforge workflow ossfuzz_campaign . --no-auto-start + +# Auto-stop worker after completion +fuzzforge workflow ossfuzz_campaign . --wait --auto-stop +``` + #### `fuzzforge workflow status [execution-id]` Check the status of a workflow execution. @@ -366,6 +425,12 @@ preferences: show_progress_bars: true table_style: "rich" color_output: true + +workers: + auto_start_workers: true # Auto-start workers when needed + auto_stop_workers: false # Auto-stop workers after completion + worker_startup_timeout: 60 # Worker startup timeout (seconds) + docker_compose_file: null # Custom docker-compose.yml path ``` ## šŸ”§ Advanced Usage diff --git a/cli/completion_install.py b/cli/completion_install.py index 3fc5dc9..bc1784d 100644 --- a/cli/completion_install.py +++ b/cli/completion_install.py @@ -207,7 +207,7 @@ def install_zsh_completion(): # Add fpath to .zshrc if not present zshrc = Path.home() / ".zshrc" - fpath_line = f'fpath=(~/.zsh/completions $fpath)' + fpath_line = 'fpath=(~/.zsh/completions $fpath)' autoload_line = 'autoload -U compinit && compinit' if zshrc.exists(): @@ -222,7 +222,7 @@ def install_zsh_completion(): if lines_to_add: with zshrc.open("a") as f: - f.write(f"\n# FuzzForge CLI completion\n") + f.write("\n# FuzzForge CLI completion\n") for line in lines_to_add: f.write(f"{line}\n") print("āœ… Added completion setup to ~/.zshrc") diff --git a/cli/main.py b/cli/main.py index f51211d..627f3f9 100644 --- a/cli/main.py +++ b/cli/main.py @@ -15,7 +15,6 @@ This module provides the main entry point for the FuzzForge CLI application. # Additional attribution and requirements are provided in the NOTICE file. -import typer from src.fuzzforge_cli.main import app if __name__ == "__main__": diff --git a/cli/pyproject.toml b/cli/pyproject.toml index 5204c72..1b8ddd9 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fuzzforge-cli" -version = "0.6.0" +version = "0.7.0" description = "FuzzForge CLI - Command-line interface for FuzzForge security testing platform" readme = "README.md" authors = [ diff --git a/cli/src/fuzzforge_cli/api_validation.py b/cli/src/fuzzforge_cli/api_validation.py index 4174947..1f9aa52 100644 --- a/cli/src/fuzzforge_cli/api_validation.py +++ b/cli/src/fuzzforge_cli/api_validation.py @@ -14,10 +14,10 @@ API response validation and graceful degradation utilities. import logging -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional from pydantic import BaseModel, ValidationError as PydanticValidationError -from .exceptions import ValidationError, APIConnectionError +from .exceptions import ValidationError logger = logging.getLogger(__name__) @@ -29,7 +29,6 @@ class WorkflowMetadata(BaseModel): author: Optional[str] = None description: Optional[str] = None parameters: Dict[str, Any] = {} - supported_volume_modes: List[str] = ["ro", "rw"] class RunStatus(BaseModel): diff --git a/cli/src/fuzzforge_cli/commands/ai.py b/cli/src/fuzzforge_cli/commands/ai.py index c30febd..a5834dd 100644 --- a/cli/src/fuzzforge_cli/commands/ai.py +++ b/cli/src/fuzzforge_cli/commands/ai.py @@ -15,15 +15,10 @@ from __future__ import annotations import asyncio import os -from datetime import datetime -from typing import Optional import typer from rich.console import Console -from rich.panel import Panel -from rich.table import Table -from ..config import ProjectConfigManager console = Console() app = typer.Typer(name="ai", help="Interact with the FuzzForge AI system") @@ -33,19 +28,12 @@ app = typer.Typer(name="ai", help="Interact with the FuzzForge AI system") def ai_agent() -> None: """Launch the full AI agent CLI with A2A orchestration.""" console.print("[cyan]šŸ¤– Opening Project FuzzForge AI Agent session[/cyan]\n") - try: from fuzzforge_ai.cli import FuzzForgeCLI - cli = FuzzForgeCLI() asyncio.run(cli.run()) except ImportError as exc: console.print(f"[red]Failed to import AI CLI:[/red] {exc}") - console.print("[dim]Ensure AI dependencies are installed (pip install -e .)[/dim]") - raise typer.Exit(1) from exc - except Exception as exc: # pragma: no cover - runtime safety - console.print(f"[red]Failed to launch AI agent:[/red] {exc}") - console.print("[dim]Check that .env contains LITELLM_MODEL and API keys[/dim]") raise typer.Exit(1) from exc @@ -53,41 +41,15 @@ def ai_agent() -> None: @app.command("status") def ai_status() -> None: """Show AI system health and configuration.""" - try: - status = asyncio.run(get_ai_status_async()) - except Exception as exc: # pragma: no cover - console.print(f"[red]Failed to get AI status:[/red] {exc}") - raise typer.Exit(1) from exc - - console.print("[bold cyan]šŸ¤– FuzzForge AI System Status[/bold cyan]\n") - - config_table = Table(title="Configuration", show_header=True, header_style="bold magenta") - config_table.add_column("Setting", style="bold") - config_table.add_column("Value", style="cyan") - config_table.add_column("Status", style="green") - - for key, info in status["config"].items(): - status_icon = "āœ…" if info["configured"] else "āŒ" - display_value = info["value"] if info["value"] else "-" - config_table.add_row(key, display_value, f"{status_icon}") - - console.print(config_table) - console.print() - - components_table = Table(title="AI Components", show_header=True, header_style="bold magenta") - components_table.add_column("Component", style="bold") - components_table.add_column("Status", style="green") - components_table.add_column("Details", style="dim") - - for component, info in status["components"].items(): - status_icon = "šŸŸ¢" if info["available"] else "šŸ”“" - components_table.add_row(component, status_icon, info["details"]) - - console.print(components_table) - - if status["agents"]: - console.print() - console.print(f"[bold green]āœ“[/bold green] {len(status['agents'])} agents registered") + # TODO: Implement AI status checking + # This command is a placeholder for future health monitoring functionality + console.print("šŸš§ [yellow]AI status command is not yet implemented.[/yellow]") + console.print("\nPlanned features:") + console.print(" ā€¢ LLM provider connectivity") + console.print(" ā€¢ API key validation") + console.print(" ā€¢ Registered agents status") + console.print(" ā€¢ Memory/session persistence health") + console.print("\nFor now, use [cyan]ff ai agent[/cyan] to launch the AI agent.") @app.command("server") diff --git a/cli/src/fuzzforge_cli/commands/config.py b/cli/src/fuzzforge_cli/commands/config.py index 3af160b..1373fd7 100644 --- a/cli/src/fuzzforge_cli/commands/config.py +++ b/cli/src/fuzzforge_cli/commands/config.py @@ -18,13 +18,11 @@ from pathlib import Path from rich.console import Console from rich.table import Table from rich.panel import Panel -from rich.prompt import Prompt, Confirm +from rich.prompt import Confirm from rich import box -from typing import Optional from ..config import ( get_project_config, - ensure_project_config, get_global_config, save_global_config, FuzzForgeConfig @@ -335,7 +333,6 @@ def edit_config( """ šŸ“ Open configuration file in default editor """ - import os import subprocess if global_config: @@ -369,7 +366,7 @@ def edit_config( try: console.print(f"šŸ“ Opening {config_type} configuration in {editor}...") subprocess.run([editor, str(config_path)], check=True) - console.print(f"āœ… Configuration file edited", style="green") + console.print("āœ… Configuration file edited", style="green") except subprocess.CalledProcessError as e: console.print(f"āŒ Failed to open editor: {e}", style="red") diff --git a/cli/src/fuzzforge_cli/commands/findings.py b/cli/src/fuzzforge_cli/commands/findings.py index c4ceff8..3adfd7d 100644 --- a/cli/src/fuzzforge_cli/commands/findings.py +++ b/cli/src/fuzzforge_cli/commands/findings.py @@ -21,18 +21,17 @@ from typing import Optional, Dict, Any, List import typer from rich.console import Console -from rich.table import Table, Column +from rich.table import Table from rich.panel import Panel from rich.syntax import Syntax -from rich.tree import Tree from rich.text import Text from rich import box from ..config import get_project_config, FuzzForgeConfig from ..database import get_project_db, ensure_project_db, FindingRecord from ..exceptions import ( - handle_error, retry_on_network_error, validate_run_id, - require_project, ValidationError, DatabaseError + retry_on_network_error, validate_run_id, + require_project, ValidationError ) from fuzzforge_sdk import FuzzForgeClient @@ -159,7 +158,7 @@ def display_findings_table(sarif_data: Dict[str, Any]): driver = tool.get("driver", {}) # Tool information - console.print(f"\nšŸ” [bold]Security Analysis Results[/bold]") + console.print("\nšŸ” [bold]Security Analysis Results[/bold]") if driver.get("name"): console.print(f"Tool: {driver.get('name')} v{driver.get('version', 'unknown')}") @@ -241,7 +240,7 @@ def display_findings_table(sarif_data: Dict[str, Any]): location_text ) - console.print(f"\nšŸ“‹ [bold]Detailed Results[/bold]") + console.print("\nšŸ“‹ [bold]Detailed Results[/bold]") if len(results) > 50: console.print(f"Showing first 50 of {len(results)} results") console.print() @@ -297,7 +296,7 @@ def findings_history( console.print(f"\nšŸ“š [bold]Findings History ({len(findings)})[/bold]\n") console.print(table) - console.print(f"\nšŸ’” Use [bold cyan]fuzzforge finding [/bold cyan] to view detailed findings") + console.print("\nšŸ’” Use [bold cyan]fuzzforge finding [/bold cyan] to view detailed findings") except Exception as e: console.print(f"āŒ Failed to get findings history: {e}", style="red") @@ -710,10 +709,10 @@ def all_findings( if show_findings: display_detailed_findings(findings, max_findings) - console.print(f"\nšŸ’” Use filters to refine results: --workflow, --severity, --since") - console.print(f"šŸ’” Show findings content: --show-findings") - console.print(f"šŸ’” Export findings: --export json --output report.json") - console.print(f"šŸ’” View specific findings: [bold cyan]fuzzforge finding [/bold cyan]") + console.print("\nšŸ’” Use filters to refine results: --workflow, --severity, --since") + console.print("šŸ’” Show findings content: --show-findings") + console.print("šŸ’” Export findings: --export json --output report.json") + console.print("šŸ’” View specific findings: [bold cyan]fuzzforge finding [/bold cyan]") except Exception as e: console.print(f"āŒ Failed to get all findings: {e}", style="red") diff --git a/cli/src/fuzzforge_cli/commands/init.py b/cli/src/fuzzforge_cli/commands/init.py index 9fec614..9a9d30a 100644 --- a/cli/src/fuzzforge_cli/commands/init.py +++ b/cli/src/fuzzforge_cli/commands/init.py @@ -200,9 +200,6 @@ def _ensure_env_file(fuzzforge_dir: Path, force: bool) -> None: console=console, ) - enable_cognee = False - cognee_url = "" - session_db_path = fuzzforge_dir / "fuzzforge_sessions.db" session_db_rel = session_db_path.relative_to(fuzzforge_dir.parent) @@ -214,7 +211,7 @@ def _ensure_env_file(fuzzforge_dir: Path, force: bool) -> None: f"LLM_MODEL={llm_model}", f"LITELLM_MODEL={llm_model}", f"OPENAI_API_KEY={api_key}", - f"FUZZFORGE_MCP_URL={os.getenv('FUZZFORGE_MCP_URL', 'http://localhost:8010/mcp')}", + "FUZZFORGE_MCP_URL=http://localhost:8010/mcp", "", "# Cognee configuration mirrors the primary LLM by default", f"LLM_COGNEE_PROVIDER={llm_provider}", diff --git a/cli/src/fuzzforge_cli/commands/monitor.py b/cli/src/fuzzforge_cli/commands/monitor.py new file mode 100644 index 0000000..b308f06 --- /dev/null +++ b/cli/src/fuzzforge_cli/commands/monitor.py @@ -0,0 +1,428 @@ +""" +Real-time monitoring and statistics commands. +""" +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + + +import time +from datetime import datetime + +import typer +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.live import Live +from rich import box + +from ..config import get_project_config, FuzzForgeConfig +from ..database import ensure_project_db, CrashRecord +from fuzzforge_sdk import FuzzForgeClient + +console = Console() +app = typer.Typer() + + +def get_client() -> FuzzForgeClient: + """Get configured FuzzForge client""" + config = get_project_config() or FuzzForgeConfig() + return FuzzForgeClient(base_url=config.get_api_url(), timeout=config.get_timeout()) + + +def format_duration(seconds: int) -> str: + """Format duration in human readable format""" + if seconds < 60: + return f"{seconds}s" + elif seconds < 3600: + return f"{seconds // 60}m {seconds % 60}s" + else: + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + return f"{hours}h {minutes}m" + + +def format_number(num: int) -> str: + """Format large numbers with K, M suffixes""" + if num >= 1000000: + return f"{num / 1000000:.1f}M" + elif num >= 1000: + return f"{num / 1000:.1f}K" + else: + return str(num) + + +@app.command("stats") +def fuzzing_stats( + run_id: str = typer.Argument(..., help="Run ID to get statistics for"), + refresh: int = typer.Option( + 5, "--refresh", "-r", + help="Refresh interval in seconds" + ), + once: bool = typer.Option( + False, "--once", + help="Show stats once and exit" + ) +): + """ + šŸ“Š Show current fuzzing statistics for a run + """ + try: + with get_client() as client: + if once: + # Show stats once + stats = client.get_fuzzing_stats(run_id) + display_stats_table(stats) + else: + # Live updating stats + console.print(f"šŸ“Š [bold]Live Fuzzing Statistics[/bold] (Run: {run_id[:12]}...)") + console.print(f"Refreshing every {refresh}s. Press Ctrl+C to stop.\n") + + with Live(auto_refresh=False, console=console) as live: + while True: + try: + # Check workflow status + run_status = client.get_run_status(run_id) + stats = client.get_fuzzing_stats(run_id) + table = create_stats_table(stats) + live.update(table, refresh=True) + + # Exit if workflow completed or failed + if getattr(run_status, 'is_completed', False) or getattr(run_status, 'is_failed', False): + final_status = getattr(run_status, 'status', 'Unknown') + if getattr(run_status, 'is_completed', False): + console.print("\nāœ… [bold green]Workflow completed[/bold green]", style="green") + else: + console.print(f"\nāš ļø [bold yellow]Workflow ended[/bold yellow] | Status: {final_status}", style="yellow") + break + + time.sleep(refresh) + except KeyboardInterrupt: + console.print("\nšŸ“Š Monitoring stopped", style="yellow") + break + + except Exception as e: + console.print(f"āŒ Failed to get fuzzing stats: {e}", style="red") + raise typer.Exit(1) + + +def display_stats_table(stats): + """Display stats in a simple table""" + table = create_stats_table(stats) + console.print(table) + + +def create_stats_table(stats) -> Panel: + """Create a rich table for fuzzing statistics""" + # Create main stats table + stats_table = Table(show_header=False, box=box.SIMPLE) + stats_table.add_column("Metric", style="bold cyan") + stats_table.add_column("Value", justify="right", style="bold white") + + stats_table.add_row("Total Executions", format_number(stats.executions)) + stats_table.add_row("Executions/sec", f"{stats.executions_per_sec:.1f}") + stats_table.add_row("Total Crashes", format_number(stats.crashes)) + stats_table.add_row("Unique Crashes", format_number(stats.unique_crashes)) + + if stats.coverage is not None and stats.coverage > 0: + stats_table.add_row("Code Coverage", f"{stats.coverage} edges") + + stats_table.add_row("Corpus Size", format_number(stats.corpus_size)) + stats_table.add_row("Elapsed Time", format_duration(stats.elapsed_time)) + + if stats.last_crash_time: + time_since_crash = datetime.now() - stats.last_crash_time + stats_table.add_row("Last Crash", f"{format_duration(int(time_since_crash.total_seconds()))} ago") + + return Panel.fit( + stats_table, + title=f"šŸ“Š Fuzzing Statistics - {stats.workflow}", + subtitle=f"Run: {stats.run_id[:12]}...", + box=box.ROUNDED + ) + + +@app.command("crashes") +def crash_reports( + run_id: str = typer.Argument(..., help="Run ID to get crash reports for"), + save: bool = typer.Option( + True, "--save/--no-save", + help="Save crashes to local database" + ), + limit: int = typer.Option( + 50, "--limit", "-l", + help="Maximum number of crashes to show" + ) +): + """ + šŸ› Display crash reports for a fuzzing run + """ + try: + with get_client() as client: + console.print(f"šŸ› Fetching crash reports for run: {run_id}") + crashes = client.get_crash_reports(run_id) + + if not crashes: + console.print("āœ… No crashes found!", style="green") + return + + # Save to database if requested + if save: + db = ensure_project_db() + for crash in crashes: + crash_record = CrashRecord( + run_id=run_id, + crash_id=crash.crash_id, + signal=crash.signal, + stack_trace=crash.stack_trace, + input_file=crash.input_file, + severity=crash.severity, + timestamp=crash.timestamp + ) + db.save_crash(crash_record) + console.print("āœ… Crashes saved to local database") + + # Display crashes + crashes_to_show = crashes[:limit] + + # Summary + severity_counts = {} + signal_counts = {} + for crash in crashes: + severity_counts[crash.severity] = severity_counts.get(crash.severity, 0) + 1 + if crash.signal: + signal_counts[crash.signal] = signal_counts.get(crash.signal, 0) + 1 + + summary_table = Table(show_header=False, box=box.SIMPLE) + summary_table.add_column("Metric", style="bold cyan") + summary_table.add_column("Value", justify="right") + + summary_table.add_row("Total Crashes", str(len(crashes))) + summary_table.add_row("Unique Signals", str(len(signal_counts))) + + for severity, count in sorted(severity_counts.items()): + summary_table.add_row(f"{severity.title()} Severity", str(count)) + + console.print( + Panel.fit( + summary_table, + title="šŸ› Crash Summary", + box=box.ROUNDED + ) + ) + + # Detailed crash table + if crashes_to_show: + crashes_table = Table(box=box.ROUNDED) + crashes_table.add_column("Crash ID", style="bold cyan") + crashes_table.add_column("Signal", justify="center") + crashes_table.add_column("Severity", justify="center") + crashes_table.add_column("Timestamp", justify="center") + crashes_table.add_column("Input File", style="dim") + + for crash in crashes_to_show: + signal_emoji = { + "SIGSEGV": "šŸ’„", + "SIGABRT": "šŸ›‘", + "SIGFPE": "šŸ§®", + "SIGILL": "āš ļø" + }.get(crash.signal or "", "šŸ›") + + severity_style = { + "high": "red", + "medium": "yellow", + "low": "green" + }.get(crash.severity.lower(), "white") + + input_display = "" + if crash.input_file: + input_display = crash.input_file.split("/")[-1] # Show just filename + + crashes_table.add_row( + crash.crash_id[:12] + "..." if len(crash.crash_id) > 15 else crash.crash_id, + f"{signal_emoji} {crash.signal or 'Unknown'}", + f"[{severity_style}]{crash.severity}[/{severity_style}]", + crash.timestamp.strftime("%H:%M:%S"), + input_display + ) + + console.print("\nšŸ› [bold]Crash Details[/bold]") + if len(crashes) > limit: + console.print(f"Showing first {limit} of {len(crashes)} crashes") + console.print() + console.print(crashes_table) + + console.print(f"\nšŸ’” Use [bold cyan]fuzzforge finding {run_id}[/bold cyan] for detailed analysis") + + except Exception as e: + console.print(f"āŒ Failed to get crash reports: {e}", style="red") + raise typer.Exit(1) + + +def _live_monitor(run_id: str, refresh: int): + """Helper for live monitoring with inline real-time display""" + with get_client() as client: + start_time = time.time() + + def render_inline_stats(run_status, stats): + """Render inline stats display (non-dashboard)""" + lines = [] + + # Header line + workflow_name = getattr(stats, 'workflow', 'unknown') + status_emoji = "šŸ”„" if not getattr(run_status, 'is_completed', False) else "āœ…" + status_color = "yellow" if not getattr(run_status, 'is_completed', False) else "green" + + lines.append(f"\n[bold cyan]šŸ“Š Live Fuzzing Monitor[/bold cyan] - {workflow_name} (Run: {run_id[:12]}...)\n") + + # Stats lines with emojis + lines.append(f" [bold]āš” Executions[/bold] {format_number(stats.executions):>8} [dim]({stats.executions_per_sec:,.1f}/sec)[/dim]") + lines.append(f" [bold]šŸ’„ Crashes[/bold] {stats.crashes:>8} [dim](unique: {stats.unique_crashes})[/dim]") + lines.append(f" [bold]šŸ“¦ Corpus[/bold] {stats.corpus_size:>8} inputs") + + if stats.coverage is not None and stats.coverage > 0: + lines.append(f" [bold]šŸ“ˆ Coverage[/bold] {stats.coverage:>8} edges") + + lines.append(f" [bold]ā±ļø Elapsed[/bold] {format_duration(stats.elapsed_time):>8}") + + # Last crash info + if stats.last_crash_time: + time_since = datetime.now() - stats.last_crash_time + crash_ago = format_duration(int(time_since.total_seconds())) + lines.append(f" [bold red]šŸ› Last Crash[/bold red] {crash_ago:>8} ago") + + # Status line + status_text = getattr(run_status, 'status', 'Unknown') + current_time = datetime.now().strftime('%H:%M:%S') + lines.append(f"\n[{status_color}]{status_emoji} Status: {status_text}[/{status_color}] | Last update: [dim]{current_time}[/dim] | Refresh: {refresh}s | [dim]Press Ctrl+C to stop[/dim]") + + return "\n".join(lines) + + # Fallback stats class + class FallbackStats: + def __init__(self, run_id): + self.run_id = run_id + self.workflow = "unknown" + self.executions = 0 + self.executions_per_sec = 0.0 + self.crashes = 0 + self.unique_crashes = 0 + self.coverage = None + self.corpus_size = 0 + self.elapsed_time = 0 + self.last_crash_time = None + + with Live(auto_refresh=False, console=console) as live: + # Initial fetch + try: + run_status = client.get_run_status(run_id) + stats = client.get_fuzzing_stats(run_id) + except Exception: + stats = FallbackStats(run_id) + run_status = type("RS", (), {"status":"Unknown","is_completed":False,"is_failed":False})() + + live.update(render_inline_stats(run_status, stats), refresh=True) + + # Polling loop + consecutive_errors = 0 + max_errors = 5 + + while True: + try: + # Poll for updates + try: + run_status = client.get_run_status(run_id) + consecutive_errors = 0 + except Exception as e: + consecutive_errors += 1 + if consecutive_errors >= max_errors: + console.print(f"\nāŒ Too many errors getting run status: {e}", style="red") + break + time.sleep(refresh) + continue + + # Try to get fuzzing stats + try: + stats = client.get_fuzzing_stats(run_id) + except Exception: + stats = FallbackStats(run_id) + + # Update display + live.update(render_inline_stats(run_status, stats), refresh=True) + + # Check if completed + if getattr(run_status, 'is_completed', False) or getattr(run_status, 'is_failed', False): + break + + # Wait before next poll + time.sleep(refresh) + + except KeyboardInterrupt: + raise + except Exception as e: + console.print(f"\nāš ļø Monitoring error: {e}", style="yellow") + time.sleep(refresh) + + # Final status + final_status = getattr(run_status, 'status', 'Unknown') + total_time = format_duration(int(time.time() - start_time)) + + if getattr(run_status, 'is_completed', False): + console.print(f"\nāœ… [bold green]Run completed successfully[/bold green] | Total runtime: {total_time}") + else: + console.print(f"\nāš ļø [bold yellow]Run ended[/bold yellow] | Status: {final_status} | Total runtime: {total_time}") + + +@app.command("live") +def live_monitor( + run_id: str = typer.Argument(..., help="Run ID to monitor live"), + refresh: int = typer.Option( + 2, "--refresh", "-r", + help="Refresh interval in seconds" + ) +): + """ + šŸ“ŗ Real-time inline monitoring with live statistics updates + """ + try: + _live_monitor(run_id, refresh) + except KeyboardInterrupt: + console.print("\n\nšŸ“Š Monitoring stopped by user.", style="yellow") + except Exception as e: + console.print(f"\nāŒ Failed to start live monitoring: {e}", style="red") + raise typer.Exit(1) + + +def create_progress_bar(percentage: float, color: str = "green") -> str: + """Create a simple text progress bar""" + width = 20 + filled = int((percentage / 100) * width) + bar = "ā–ˆ" * filled + "ā–‘" * (width - filled) + return f"[{color}]{bar}[/{color}] {percentage:.1f}%" + + +@app.callback(invoke_without_command=True) +def monitor_callback(ctx: typer.Context): + """ + šŸ“Š Real-time monitoring and statistics + """ + # Check if a subcommand is being invoked + if ctx.invoked_subcommand is not None: + # Let the subcommand handle it + return + + # Show help message for default command + from rich.console import Console + console = Console() + console.print("šŸ“Š [bold cyan]Monitor Command[/bold cyan]") + console.print("\nAvailable subcommands:") + console.print(" ā€¢ [cyan]ff monitor stats [/cyan] - Show execution statistics") + console.print(" ā€¢ [cyan]ff monitor crashes [/cyan] - Show crash reports") + console.print(" ā€¢ [cyan]ff monitor live [/cyan] - Real-time inline monitoring") diff --git a/cli/src/fuzzforge_cli/commands/status.py b/cli/src/fuzzforge_cli/commands/status.py index 4874179..5d78042 100644 --- a/cli/src/fuzzforge_cli/commands/status.py +++ b/cli/src/fuzzforge_cli/commands/status.py @@ -115,7 +115,7 @@ def show_status(): api_table.add_column("Property", style="bold cyan") api_table.add_column("Value") - api_table.add_row("Status", f"āœ… Connected") + api_table.add_row("Status", "āœ… Connected") api_table.add_row("Service", f"{api_status.name} v{api_status.version}") api_table.add_row("Workflows", str(len(workflows))) diff --git a/cli/src/fuzzforge_cli/commands/workflow_exec.py b/cli/src/fuzzforge_cli/commands/workflow_exec.py index 5647f20..959e94f 100644 --- a/cli/src/fuzzforge_cli/commands/workflow_exec.py +++ b/cli/src/fuzzforge_cli/commands/workflow_exec.py @@ -13,47 +13,37 @@ Replaces the old 'runs' terminology with cleaner workflow-centric commands. # # Additional attribution and requirements are provided in the NOTICE file. + +import json import time from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Optional, Dict, Any, List import typer -from fuzzforge_sdk import FuzzForgeClient, WorkflowSubmission -from rich import box from rich.console import Console -from rich.panel import Panel -from rich.prompt import Confirm, Prompt from rich.table import Table +from rich.panel import Panel +from rich.prompt import Prompt, Confirm +from rich import box -from ..config import FuzzForgeConfig, get_project_config -from ..constants import ( - DEFAULT_VOLUME_MODE, - MAX_RETRIES, - MAX_RUN_ID_DISPLAY_LENGTH, - POLL_INTERVAL, - PROGRESS_STEP_DELAYS, - RETRY_DELAY, - STATUS_EMOJIS, -) -from ..database import RunRecord, ensure_project_db, get_project_db +from ..config import get_project_config, FuzzForgeConfig +from ..database import get_project_db, ensure_project_db, RunRecord from ..exceptions import ( - DatabaseError, - ValidationError, - handle_error, - require_project, - retry_on_network_error, - safe_json_load, + handle_error, retry_on_network_error, safe_json_load, require_project, + ValidationError, DatabaseError +) +from ..validation import ( + validate_run_id, validate_workflow_name, validate_target_path, + validate_parameters, validate_timeout ) from ..progress import step_progress -from ..validation import ( - validate_parameters, - validate_run_id, - validate_target_path, - validate_timeout, - validate_volume_mode, - validate_workflow_name, +from ..constants import ( + STATUS_EMOJIS, MAX_RUN_ID_DISPLAY_LENGTH, DEFAULT_VOLUME_MODE, + PROGRESS_STEP_DELAYS, MAX_RETRIES, RETRY_DELAY, POLL_INTERVAL ) +from ..worker_manager import WorkerManager +from fuzzforge_sdk import FuzzForgeClient, WorkflowSubmission console = Console() app = typer.Typer() @@ -71,6 +61,47 @@ def status_emoji(status: str) -> str: return STATUS_EMOJIS.get(status.lower(), STATUS_EMOJIS["unknown"]) +def should_fail_build(sarif_data: Dict[str, Any], fail_on: str) -> bool: + """ + Check if findings warrant build failure based on SARIF severity levels. + + Args: + sarif_data: SARIF format findings data + fail_on: Comma-separated SARIF levels (error,warning,note,info,all,none) + + Returns: + True if build should fail, False otherwise + """ + if fail_on == "none": + return False + + # Parse fail_on parameter - accept SARIF levels + if fail_on == "all": + check_levels = {"error", "warning", "note", "info"} + else: + check_levels = {s.strip().lower() for s in fail_on.split(",")} + + # Validate levels + valid_levels = {"error", "warning", "note", "info", "none"} + invalid = check_levels - valid_levels + if invalid: + console.print(f"āš ļø Invalid SARIF levels: {', '.join(invalid)}", style="yellow") + console.print("Valid levels: error, warning, note, info, all, none") + + # Check SARIF results + runs = sarif_data.get("runs", []) + if not runs: + return False + + results = runs[0].get("results", []) + for result in results: + level = result.get("level", "note") # SARIF default is "note" + if level in check_levels: + return True + + return False + + def parse_inline_parameters(params: List[str]) -> Dict[str, Any]: """Parse inline key=value parameters using improved validation""" return validate_parameters(params) @@ -83,26 +114,22 @@ def execute_workflow_submission( parameters: Dict[str, Any], volume_mode: str, timeout: Optional[int], - interactive: bool, + interactive: bool ) -> Any: - """Handle the workflow submission process""" + """Handle the workflow submission process with file upload""" # Get workflow metadata for parameter validation console.print(f"šŸ”§ Getting workflow information for: {workflow}") workflow_meta = client.get_workflow_metadata(workflow) - param_response = client.get_workflow_parameters(workflow) # Interactive parameter input if interactive and workflow_meta.parameters.get("properties"): properties = workflow_meta.parameters.get("properties", {}) required_params = set(workflow_meta.parameters.get("required", [])) - defaults = param_response.defaults missing_required = required_params - set(parameters.keys()) if missing_required: - console.print( - f"\nšŸ“ [bold]Missing required parameters:[/bold] {', '.join(missing_required)}" - ) + console.print(f"\nšŸ“ [bold]Missing required parameters:[/bold] {', '.join(missing_required)}") console.print("Please provide values:\n") for param_name in missing_required: @@ -124,40 +151,16 @@ def execute_workflow_submission( elif param_type == "number": parameters[param_name] = float(user_input) elif param_type == "boolean": - parameters[param_name] = user_input.lower() in ( - "true", - "yes", - "1", - "on", - ) + parameters[param_name] = user_input.lower() in ("true", "yes", "1", "on") elif param_type == "array": - parameters[param_name] = [ - item.strip() - for item in user_input.split(",") - if item.strip() - ] + parameters[param_name] = [item.strip() for item in user_input.split(",") if item.strip()] else: parameters[param_name] = user_input break except ValueError as e: console.print(f"āŒ Invalid {param_type}: {e}", style="red") - # Validate volume mode - validate_volume_mode(volume_mode) - if volume_mode not in workflow_meta.supported_volume_modes: - raise ValidationError( - "volume mode", - volume_mode, - f"one of: {', '.join(workflow_meta.supported_volume_modes)}", - ) - - # Create submission - submission = WorkflowSubmission( - target_path=target_path, - volume_mode=volume_mode, - parameters=parameters, - timeout=timeout, - ) + # Note: volume_mode is no longer used (Temporal uses MinIO storage) # Show submission summary console.print("\nšŸŽÆ [bold]Executing workflow:[/bold]") @@ -169,6 +172,22 @@ def execute_workflow_submission( if timeout: console.print(f" Timeout: {timeout}s") + # Check if target path exists locally + target_path_obj = Path(target_path) + use_upload = target_path_obj.exists() + + if use_upload: + # Show file/directory info + if target_path_obj.is_dir(): + num_files = sum(1 for _ in target_path_obj.rglob("*") if _.is_file()) + console.print(f" Upload: Directory with {num_files} files") + else: + size_mb = target_path_obj.stat().st_size / (1024 * 1024) + console.print(f" Upload: File ({size_mb:.2f} MB)") + else: + console.print(" [yellow]āš ļø Warning: Target path does not exist locally[/yellow]") + console.print(" [yellow] Attempting to use path-based submission (backend must have access)[/yellow]") + # Only ask for confirmation in interactive mode if interactive: if not Confirm.ask("\nExecute workflow?", default=True, console=console): @@ -180,72 +199,132 @@ def execute_workflow_submission( # Submit the workflow with enhanced progress console.print(f"\nšŸš€ Executing workflow: [bold yellow]{workflow}[/bold yellow]") - steps = [ - "Validating workflow configuration", - "Connecting to FuzzForge API", - "Uploading parameters and settings", - "Creating workflow deployment", - "Initializing execution environment", - ] + if use_upload: + # Use new upload-based submission + steps = [ + "Validating workflow configuration", + "Creating tarball (if directory)", + "Uploading target to backend", + "Starting workflow execution", + "Initializing execution environment" + ] - with step_progress(steps, f"Executing {workflow}") as progress: - progress.next_step() # Validating - time.sleep(PROGRESS_STEP_DELAYS["validating"]) + with step_progress(steps, f"Executing {workflow}") as progress: + progress.next_step() # Validating + time.sleep(PROGRESS_STEP_DELAYS["validating"]) - progress.next_step() # Connecting - time.sleep(PROGRESS_STEP_DELAYS["connecting"]) + progress.next_step() # Creating tarball + time.sleep(PROGRESS_STEP_DELAYS["connecting"]) - progress.next_step() # Uploading - response = client.submit_workflow(workflow, submission) - time.sleep(PROGRESS_STEP_DELAYS["uploading"]) + progress.next_step() # Uploading + # Use the new upload method + response = client.submit_workflow_with_upload( + workflow_name=workflow, + target_path=target_path, + parameters=parameters, + timeout=timeout + ) + time.sleep(PROGRESS_STEP_DELAYS["uploading"]) - progress.next_step() # Creating deployment - time.sleep(PROGRESS_STEP_DELAYS["creating"]) + progress.next_step() # Starting + time.sleep(PROGRESS_STEP_DELAYS["creating"]) - progress.next_step() # Initializing - time.sleep(PROGRESS_STEP_DELAYS["initializing"]) + progress.next_step() # Initializing + time.sleep(PROGRESS_STEP_DELAYS["initializing"]) - progress.complete("Workflow started successfully!") + progress.complete("Workflow started successfully!") + else: + # Fall back to path-based submission (for backward compatibility) + steps = [ + "Validating workflow configuration", + "Connecting to FuzzForge API", + "Submitting workflow parameters", + "Creating workflow deployment", + "Initializing execution environment" + ] + + with step_progress(steps, f"Executing {workflow}") as progress: + progress.next_step() # Validating + time.sleep(PROGRESS_STEP_DELAYS["validating"]) + + progress.next_step() # Connecting + time.sleep(PROGRESS_STEP_DELAYS["connecting"]) + + progress.next_step() # Submitting + submission = WorkflowSubmission( + target_path=target_path, + volume_mode=volume_mode, + parameters=parameters, + timeout=timeout + ) + response = client.submit_workflow(workflow, submission) + time.sleep(PROGRESS_STEP_DELAYS["uploading"]) + + progress.next_step() # Creating deployment + time.sleep(PROGRESS_STEP_DELAYS["creating"]) + + progress.next_step() # Initializing + time.sleep(PROGRESS_STEP_DELAYS["initializing"]) + + progress.complete("Workflow started successfully!") return response # Main workflow execution command (replaces 'runs submit') -@app.command( - name="exec", hidden=True -) # Hidden because it will be called from main workflow command +@app.command(name="exec", hidden=True) # Hidden because it will be called from main workflow command def execute_workflow( workflow: str = typer.Argument(..., help="Workflow name to execute"), target_path: str = typer.Argument(..., help="Path to analyze"), - params: List[str] = typer.Argument( - default=None, help="Parameters as key=value pairs" - ), + params: List[str] = typer.Argument(default=None, help="Parameters as key=value pairs"), param_file: Optional[str] = typer.Option( - None, "--param-file", "-f", help="JSON file containing workflow parameters" + None, "--param-file", "-f", + help="JSON file containing workflow parameters" ), volume_mode: str = typer.Option( - DEFAULT_VOLUME_MODE, - "--volume-mode", - "-v", - help="Volume mount mode: ro (read-only) or rw (read-write)", + DEFAULT_VOLUME_MODE, "--volume-mode", "-v", + help="Volume mount mode: ro (read-only) or rw (read-write)" ), timeout: Optional[int] = typer.Option( - None, "--timeout", "-t", help="Execution timeout in seconds" + None, "--timeout", "-t", + help="Execution timeout in seconds" ), interactive: bool = typer.Option( - True, - "--interactive/--no-interactive", - "-i/-n", - help="Interactive parameter input for missing required parameters", + True, "--interactive/--no-interactive", "-i/-n", + help="Interactive parameter input for missing required parameters" ), wait: bool = typer.Option( - False, "--wait", "-w", help="Wait for execution to complete" + False, "--wait", "-w", + help="Wait for execution to complete" ), + live: bool = typer.Option( + False, "--live", "-l", + help="Start live monitoring after execution (useful for fuzzing workflows)" + ), + auto_start: Optional[bool] = typer.Option( + None, "--auto-start/--no-auto-start", + help="Automatically start required worker if not running (default: from config)" + ), + auto_stop: Optional[bool] = typer.Option( + None, "--auto-stop/--no-auto-stop", + help="Automatically stop worker after execution completes (default: from config)" + ), + fail_on: Optional[str] = typer.Option( + None, "--fail-on", + help="Fail build if findings match severity (critical,high,medium,low,all,none). Use with --wait" + ), + export_sarif: Optional[str] = typer.Option( + None, "--export-sarif", + help="Export SARIF results to file after completion. Use with --wait" + ) ): """ šŸš€ Execute a workflow on a target + Use --live for fuzzing workflows to see real-time progress. Use --wait to wait for completion without live dashboard. + Use --fail-on with --wait to fail CI builds based on finding severity. + Use --export-sarif with --wait to export SARIF findings to a file. """ try: # Validate inputs @@ -281,23 +360,63 @@ def execute_workflow( except Exception as e: handle_error(e, "parsing parameters") + # Get config for worker management settings + config = get_project_config() or FuzzForgeConfig() + should_auto_start = auto_start if auto_start is not None else config.workers.auto_start_workers + should_auto_stop = auto_stop if auto_stop is not None else config.workers.auto_stop_workers + + worker_container = None # Track for cleanup + worker_mgr = None + wait_completed = False # Track if wait completed successfully + try: with get_client() as client: + # Get worker information for this workflow + try: + console.print(f"šŸ” Checking worker requirements for: {workflow}") + worker_info = client.get_workflow_worker_info(workflow) + + # Initialize worker manager + compose_file = config.workers.docker_compose_file + worker_mgr = WorkerManager( + compose_file=Path(compose_file) if compose_file else None, + startup_timeout=config.workers.worker_startup_timeout + ) + + # Ensure worker is running + worker_container = worker_info["worker_container"] + worker_service = worker_info.get("worker_service", f"worker-{worker_info['vertical']}") + if not worker_mgr.ensure_worker_running(worker_info, auto_start=should_auto_start): + console.print( + f"āŒ Worker not available: {worker_info['vertical']}", + style="red" + ) + console.print( + f"šŸ’” Start the worker manually: docker compose up -d {worker_service}" + ) + raise typer.Exit(1) + + except typer.Exit: + raise # Re-raise Exit to preserve exit code + except Exception as e: + # If we can't get worker info, warn but continue (might be old backend) + console.print( + f"āš ļø Could not check worker requirements: {e}", + style="yellow" + ) + console.print( + " Continuing without worker management...", + style="yellow" + ) + response = execute_workflow_submission( - client, - workflow, - target_path, - parameters, - volume_mode, - timeout, - interactive, + client, workflow, target_path, parameters, + volume_mode, timeout, interactive ) console.print("āœ… Workflow execution started!", style="green") console.print(f" Execution ID: [bold cyan]{response.run_id}[/bold cyan]") - console.print( - f" Status: {status_emoji(response.status)} {response.status}" - ) + console.print(f" Status: {status_emoji(response.status)} {response.status}") # Save to database try: @@ -308,67 +427,121 @@ def execute_workflow( status=response.status, target_path=target_path, parameters=parameters, - created_at=datetime.now(), + created_at=datetime.now() ) db.save_run(run_record) except Exception as e: # Don't fail the whole operation if database save fails - console.print( - f"āš ļø Failed to save execution to database: {e}", style="yellow" - ) + console.print(f"āš ļø Failed to save execution to database: {e}", style="yellow") - console.print( - f"šŸ’” Check status: [bold cyan]fuzzforge workflow status {response.run_id}[/bold cyan]" - ) + console.print(f"\nšŸ’” Monitor progress: [bold cyan]fuzzforge monitor stats {response.run_id}[/bold cyan]") + console.print(f"šŸ’” Check status: [bold cyan]fuzzforge workflow status {response.run_id}[/bold cyan]") + + # Suggest --live for fuzzing workflows + if not live and not wait and "fuzzing" in workflow.lower(): + console.print(f"šŸ’” Next time try: [bold cyan]fuzzforge workflow {workflow} {target_path} --live[/bold cyan] for real-time monitoring", style="dim") + + # Start live monitoring if requested + if live: + # Check if this is a fuzzing workflow to show appropriate messaging + is_fuzzing = "fuzzing" in workflow.lower() + if is_fuzzing: + console.print("\nšŸ“ŗ Starting live fuzzing monitor...") + console.print("šŸ’” You'll see real-time crash discovery, execution stats, and coverage data.") + else: + console.print("\nšŸ“ŗ Starting live monitoring...") + + console.print("Press Ctrl+C to stop monitoring (execution continues in background).\n") + + try: + from ..commands.monitor import live_monitor + # Import monitor command and run it + live_monitor(response.run_id, refresh=3) + except KeyboardInterrupt: + console.print("\nā¹ļø Live monitoring stopped (execution continues in background)", style="yellow") + except Exception as e: + console.print(f"āš ļø Failed to start live monitoring: {e}", style="yellow") + console.print(f"šŸ’” You can still monitor manually: [bold cyan]fuzzforge monitor {response.run_id}[/bold cyan]") # Wait for completion if requested - if wait: + elif wait: console.print("\nā³ Waiting for execution to complete...") try: - final_status = client.wait_for_completion( - response.run_id, poll_interval=POLL_INTERVAL - ) + final_status = client.wait_for_completion(response.run_id, poll_interval=POLL_INTERVAL) # Update database try: db.update_run_status( response.run_id, final_status.status, - completed_at=datetime.now() - if final_status.is_completed - else None, + completed_at=datetime.now() if final_status.is_completed else None ) except Exception as e: - console.print( - f"āš ļø Failed to update database: {e}", style="yellow" - ) + console.print(f"āš ļø Failed to update database: {e}", style="yellow") - console.print( - f"šŸ Execution completed with status: {status_emoji(final_status.status)} {final_status.status}" - ) + console.print(f"šŸ Execution completed with status: {status_emoji(final_status.status)} {final_status.status}") + wait_completed = True # Mark wait as completed if final_status.is_completed: - console.print( - f"šŸ’” View findings: [bold cyan]fuzzforge findings {response.run_id}[/bold cyan]" - ) + # Export SARIF if requested + if export_sarif: + try: + console.print("\nšŸ“¤ Exporting SARIF results...") + findings = client.get_run_findings(response.run_id) + output_path = Path(export_sarif) + with open(output_path, 'w') as f: + json.dump(findings.sarif, f, indent=2) + console.print(f"āœ… SARIF exported to: [bold cyan]{output_path}[/bold cyan]") + except Exception as e: + console.print(f"āš ļø Failed to export SARIF: {e}", style="yellow") + + # Check if build should fail based on findings + if fail_on: + try: + console.print(f"\nšŸ” Checking findings against severity threshold: {fail_on}") + findings = client.get_run_findings(response.run_id) + if should_fail_build(findings.sarif, fail_on): + console.print("āŒ [bold red]Build failed: Found blocking security issues[/bold red]") + console.print(f"šŸ’” View details: [bold cyan]fuzzforge finding {response.run_id}[/bold cyan]") + raise typer.Exit(1) + else: + console.print("āœ… [bold green]No blocking security issues found[/bold green]") + except typer.Exit: + raise # Re-raise Exit to preserve exit code + except Exception as e: + console.print(f"āš ļø Failed to check findings: {e}", style="yellow") + + if not fail_on and not export_sarif: + console.print(f"šŸ’” View findings: [bold cyan]fuzzforge findings {response.run_id}[/bold cyan]") except KeyboardInterrupt: - console.print( - "\nā¹ļø Monitoring cancelled (execution continues in background)", - style="yellow", - ) + console.print("\nā¹ļø Monitoring cancelled (execution continues in background)", style="yellow") + except typer.Exit: + raise # Re-raise Exit to preserve exit code except Exception as e: handle_error(e, "waiting for completion") + except typer.Exit: + raise # Re-raise Exit to preserve exit code except Exception as e: handle_error(e, "executing workflow") + finally: + # Stop worker if auto-stop is enabled and wait completed + if should_auto_stop and worker_container and worker_mgr and wait_completed: + try: + console.print("\nšŸ›‘ Stopping worker (auto-stop enabled)...") + if worker_mgr.stop_worker(worker_container): + console.print(f"āœ… Worker stopped: {worker_container}") + except Exception as e: + console.print( + f"āš ļø Failed to stop worker: {e}", + style="yellow" + ) @app.command("status") def workflow_status( - execution_id: Optional[str] = typer.Argument( - None, help="Execution ID to check (defaults to most recent)" - ), + execution_id: Optional[str] = typer.Argument(None, help="Execution ID to check (defaults to most recent)") ): """ šŸ“Š Check the status of a workflow execution @@ -387,9 +560,7 @@ def workflow_status( if not execution_id: recent_runs = db.list_runs(limit=1) if not recent_runs: - console.print( - "āš ļø No executions found in project database", style="yellow" - ) + console.print("āš ļø No executions found in project database", style="yellow") raise typer.Exit(0) execution_id = recent_runs[0].run_id console.print(f"šŸ” Using most recent execution: {execution_id}") @@ -405,7 +576,7 @@ def workflow_status( db.update_run_status( execution_id, status.status, - completed_at=status.updated_at if status.is_completed else None, + completed_at=status.updated_at if status.is_completed else None ) except Exception as e: console.print(f"āš ļø Failed to update database: {e}", style="yellow") @@ -425,24 +596,23 @@ def workflow_status( if status.is_completed: duration = status.updated_at - status.created_at - status_table.add_row( - "Duration", str(duration).split(".")[0] - ) # Remove microseconds + status_table.add_row("Duration", str(duration).split('.')[0]) # Remove microseconds console.print( - Panel.fit(status_table, title="šŸ“Š Status Information", box=box.ROUNDED) + Panel.fit( + status_table, + title="šŸ“Š Status Information", + box=box.ROUNDED + ) ) # Show next steps - - if status.is_completed: - console.print( - f"šŸ’” View findings: [bold cyan]fuzzforge finding {execution_id}[/bold cyan]" - ) + if status.is_running: + console.print(f"\nšŸ’” Monitor live: [bold cyan]fuzzforge monitor {execution_id}[/bold cyan]") + elif status.is_completed: + console.print(f"šŸ’” View findings: [bold cyan]fuzzforge finding {execution_id}[/bold cyan]") elif status.is_failed: - console.print( - f"šŸ’” Check logs: [bold cyan]fuzzforge workflow logs {execution_id}[/bold cyan]" - ) + console.print(f"šŸ’” Check logs: [bold cyan]fuzzforge workflow logs {execution_id}[/bold cyan]") except Exception as e: handle_error(e, "getting execution status") @@ -450,15 +620,9 @@ def workflow_status( @app.command("history") def workflow_history( - workflow: Optional[str] = typer.Option( - None, "--workflow", "-w", help="Filter by workflow name" - ), - status: Optional[str] = typer.Option( - None, "--status", "-s", help="Filter by status" - ), - limit: int = typer.Option( - 20, "--limit", "-l", help="Maximum number of executions to show" - ), + workflow: Optional[str] = typer.Option(None, "--workflow", "-w", help="Filter by workflow name"), + status: Optional[str] = typer.Option(None, "--status", "-s", help="Filter by status"), + limit: int = typer.Option(20, "--limit", "-l", help="Maximum number of executions to show") ): """ šŸ“‹ Show workflow execution history @@ -491,14 +655,12 @@ def workflow_history( param_str = f"{param_count} params" if param_count > 0 else "-" table.add_row( - run.run_id[:12] + "..." - if len(run.run_id) > MAX_RUN_ID_DISPLAY_LENGTH - else run.run_id, + run.run_id[:12] + "..." if len(run.run_id) > MAX_RUN_ID_DISPLAY_LENGTH else run.run_id, run.workflow, f"{status_emoji(run.status)} {run.status}", Path(run.target_path).name, run.created_at.strftime("%m-%d %H:%M"), - param_str, + param_str ) console.print(f"\nšŸ“‹ [bold]Workflow Execution History ({len(runs)})[/bold]") @@ -509,9 +671,7 @@ def workflow_history( console.print() console.print(table) - console.print( - "\nšŸ’” Use [bold cyan]fuzzforge workflow status [/bold cyan] for detailed status" - ) + console.print("\nšŸ’” Use [bold cyan]fuzzforge workflow status [/bold cyan] for detailed status") except Exception as e: handle_error(e, "listing execution history") @@ -519,15 +679,11 @@ def workflow_history( @app.command("retry") def retry_workflow( - execution_id: Optional[str] = typer.Argument( - None, help="Execution ID to retry (defaults to most recent)" - ), + execution_id: Optional[str] = typer.Argument(None, help="Execution ID to retry (defaults to most recent)"), modify_params: bool = typer.Option( - False, - "--modify-params", - "-m", - help="Interactively modify parameters before retrying", - ), + False, "--modify-params", "-m", + help="Interactively modify parameters before retrying" + ) ): """ šŸ”„ Retry a workflow execution with the same or modified parameters @@ -553,9 +709,7 @@ def retry_workflow( # Get original execution original_run = db.get_run(execution_id) if not original_run: - raise ValidationError( - "execution_id", execution_id, "an existing execution ID in the database" - ) + raise ValidationError("execution_id", execution_id, "an existing execution ID in the database") console.print(f"šŸ”„ [bold]Retrying workflow:[/bold] {original_run.workflow}") console.print(f" Original Execution ID: {execution_id}") @@ -567,27 +721,22 @@ def retry_workflow( if modify_params and parameters: console.print("\nšŸ“ [bold]Current parameters:[/bold]") for key, value in parameters.items(): - new_value = Prompt.ask(f"{key}", default=str(value), console=console) + new_value = Prompt.ask( + f"{key}", + default=str(value), + console=console + ) if new_value != str(value): # Try to maintain type try: if isinstance(value, bool): - parameters[key] = new_value.lower() in ( - "true", - "yes", - "1", - "on", - ) + parameters[key] = new_value.lower() in ("true", "yes", "1", "on") elif isinstance(value, int): parameters[key] = int(new_value) elif isinstance(value, float): parameters[key] = float(new_value) elif isinstance(value, list): - parameters[key] = [ - item.strip() - for item in new_value.split(",") - if item.strip() - ] + parameters[key] = [item.strip() for item in new_value.split(",") if item.strip()] else: parameters[key] = new_value except ValueError: @@ -596,18 +745,15 @@ def retry_workflow( # Submit new execution with get_client() as client: submission = WorkflowSubmission( - target_path=original_run.target_path, parameters=parameters + target_path=original_run.target_path, + parameters=parameters ) response = client.submit_workflow(original_run.workflow, submission) console.print("\nāœ… Retry submitted successfully!", style="green") - console.print( - f" New Execution ID: [bold cyan]{response.run_id}[/bold cyan]" - ) - console.print( - f" Status: {status_emoji(response.status)} {response.status}" - ) + console.print(f" New Execution ID: [bold cyan]{response.run_id}[/bold cyan]") + console.print(f" Status: {status_emoji(response.status)} {response.status}") # Save to database try: @@ -618,17 +764,13 @@ def retry_workflow( target_path=original_run.target_path, parameters=parameters, created_at=datetime.now(), - metadata={"retry_of": execution_id}, + metadata={"retry_of": execution_id} ) db.save_run(run_record) except Exception as e: - console.print( - f"āš ļø Failed to save execution to database: {e}", style="yellow" - ) + console.print(f"āš ļø Failed to save execution to database: {e}", style="yellow") - console.print( - f"\nšŸ’” Monitor progress: [bold cyan]fuzzforge monitor {response.run_id}[/bold cyan]" - ) + console.print(f"\nšŸ’” Monitor progress: [bold cyan]fuzzforge monitor stats {response.run_id}[/bold cyan]") except Exception as e: handle_error(e, "retrying workflow") @@ -638,4 +780,4 @@ def retry_workflow( def workflow_exec_callback(): """ šŸš€ Workflow execution management - """ + """ \ No newline at end of file diff --git a/cli/src/fuzzforge_cli/commands/workflows.py b/cli/src/fuzzforge_cli/commands/workflows.py index cbdd96f..e38d247 100644 --- a/cli/src/fuzzforge_cli/commands/workflows.py +++ b/cli/src/fuzzforge_cli/commands/workflows.py @@ -18,10 +18,10 @@ import typer from rich.console import Console from rich.table import Table from rich.panel import Panel -from rich.prompt import Prompt, Confirm +from rich.prompt import Prompt from rich.syntax import Syntax from rich import box -from typing import Optional, Dict, Any +from typing import Optional from ..config import get_project_config, FuzzForgeConfig from ..fuzzy import enhanced_workflow_not_found_handler @@ -68,7 +68,7 @@ def list_workflows(): console.print(f"\nšŸ”§ [bold]Available Workflows ({len(workflows)})[/bold]\n") console.print(table) - console.print(f"\nšŸ’” Use [bold cyan]fuzzforge workflows info [/bold cyan] for detailed information") + console.print("\nšŸ’” Use [bold cyan]fuzzforge workflows info [/bold cyan] for detailed information") except Exception as e: console.print(f"āŒ Failed to fetch workflows: {e}", style="red") @@ -100,7 +100,6 @@ def workflow_info( info_table.add_row("Author", workflow.author) if workflow.tags: info_table.add_row("Tags", ", ".join(workflow.tags)) - info_table.add_row("Volume Modes", ", ".join(workflow.supported_volume_modes)) info_table.add_row("Custom Docker", "āœ… Yes" if workflow.has_custom_docker else "āŒ No") console.print( @@ -193,7 +192,7 @@ def workflow_parameters( parameters = {} properties = workflow.parameters.get("properties", {}) required_params = set(workflow.parameters.get("required", [])) - defaults = param_response.defaults + defaults = param_response.default_parameters if interactive: console.print("šŸ”§ Enter parameter values (press Enter for default):\n") diff --git a/cli/src/fuzzforge_cli/completion.py b/cli/src/fuzzforge_cli/completion.py index 58aad6b..bd717cd 100644 --- a/cli/src/fuzzforge_cli/completion.py +++ b/cli/src/fuzzforge_cli/completion.py @@ -16,7 +16,7 @@ Provides intelligent tab completion for commands, workflows, run IDs, and parame import typer -from typing import List, Optional +from typing import List from pathlib import Path from .config import get_project_config, FuzzForgeConfig diff --git a/cli/src/fuzzforge_cli/config.py b/cli/src/fuzzforge_cli/config.py index ba67c9e..f21b87d 100644 --- a/cli/src/fuzzforge_cli/config.py +++ b/cli/src/fuzzforge_cli/config.py @@ -66,6 +66,15 @@ class PreferencesConfig(BaseModel): color_output: bool = True +class WorkerConfig(BaseModel): + """Worker lifecycle management configuration.""" + + auto_start_workers: bool = True + auto_stop_workers: bool = False + worker_startup_timeout: int = 60 + docker_compose_file: Optional[str] = None + + class CogneeConfig(BaseModel): """Cognee integration metadata.""" @@ -84,6 +93,7 @@ class FuzzForgeConfig(BaseModel): project: ProjectConfig = Field(default_factory=ProjectConfig) retention: RetentionConfig = Field(default_factory=RetentionConfig) preferences: PreferencesConfig = Field(default_factory=PreferencesConfig) + workers: WorkerConfig = Field(default_factory=WorkerConfig) cognee: CogneeConfig = Field(default_factory=CogneeConfig) @classmethod @@ -393,7 +403,7 @@ class ProjectConfigManager: if max_tokens: os.environ["LLM_MAX_TOKENS"] = str(max_tokens) - # Provide a default MCP endpoint for local FuzzForge backend access when unset + # FuzzForge MCP backend connection - fallback if not in .env if not os.getenv("FUZZFORGE_MCP_URL"): os.environ["FUZZFORGE_MCP_URL"] = os.getenv( "FUZZFORGE_DEFAULT_MCP_URL", diff --git a/cli/src/fuzzforge_cli/database.py b/cli/src/fuzzforge_cli/database.py index 2f488fe..3c8e86c 100644 --- a/cli/src/fuzzforge_cli/database.py +++ b/cli/src/fuzzforge_cli/database.py @@ -152,7 +152,7 @@ class FuzzForgeDatabase: if conn: try: conn.rollback() - except: + except Exception: pass # Connection might be broken if "database is locked" in str(e).lower(): raise sqlite3.OperationalError( @@ -163,18 +163,18 @@ class FuzzForgeDatabase: "Database is corrupted. Use 'ff init --force' to reset." ) from e raise - except Exception as e: + except Exception: if conn: try: conn.rollback() - except: + except Exception: pass # Connection might be broken raise finally: if conn: try: conn.close() - except: + except Exception: pass # Ensure cleanup even if close fails # Run management methods diff --git a/cli/src/fuzzforge_cli/exceptions.py b/cli/src/fuzzforge_cli/exceptions.py index b59e30d..d1137f3 100644 --- a/cli/src/fuzzforge_cli/exceptions.py +++ b/cli/src/fuzzforge_cli/exceptions.py @@ -15,7 +15,7 @@ Enhanced exception handling and error utilities for FuzzForge CLI with rich cont import time import functools -from typing import Any, Callable, Optional, Type, Union, List +from typing import Any, Callable, Optional, Union, List from pathlib import Path import typer @@ -24,20 +24,10 @@ from rich.console import Console from rich.panel import Panel from rich.text import Text from rich.table import Table -from rich.columns import Columns -from rich.syntax import Syntax -from rich.markdown import Markdown # Import SDK exceptions for rich handling from fuzzforge_sdk.exceptions import ( - FuzzForgeError as SDKFuzzForgeError, - FuzzForgeHTTPError, - DeploymentError, - WorkflowExecutionError, - ContainerError, - VolumeError, - ValidationError as SDKValidationError, - ConnectionError as SDKConnectionError + FuzzForgeError as SDKFuzzForgeError ) console = Console() @@ -269,12 +259,6 @@ def handle_error(error: Exception, context: str = "") -> None: if hasattr(error, 'context') and error.context: ctx = error.context - # Container diagnostics - if ctx.container_diagnostics: - console.print("\n[bold]Container Diagnostics:[/bold]") - display_container_diagnostics(ctx.container_diagnostics) - display_container_logs(ctx.container_diagnostics) - # Error patterns if ctx.error_patterns: display_error_patterns(ctx.error_patterns) @@ -335,7 +319,7 @@ def handle_error(error: Exception, context: str = "") -> None: # Show error details for debugging console.print(f"\n[dim yellow]Error type: {type(error).__name__}[/dim yellow]") - console.print(f"[dim yellow]Please report this issue if it persists[/dim yellow]") + console.print("[dim yellow]Please report this issue if it persists[/dim yellow]") console.print() raise typer.Exit(1) @@ -430,8 +414,9 @@ def validate_run_id(run_id: str) -> str: if not run_id or len(run_id) < 8: raise ValidationError("run_id", run_id, "at least 8 characters") - if not run_id.replace('-', '').isalnum(): - raise ValidationError("run_id", run_id, "alphanumeric characters and hyphens only") + # Allow alphanumeric characters, hyphens, and underscores + if not run_id.replace('-', '').replace('_', '').isalnum(): + raise ValidationError("run_id", run_id, "alphanumeric characters, hyphens, and underscores only") return run_id diff --git a/cli/src/fuzzforge_cli/main.py b/cli/src/fuzzforge_cli/main.py index a2e408d..5726275 100644 --- a/cli/src/fuzzforge_cli/main.py +++ b/cli/src/fuzzforge_cli/main.py @@ -12,23 +12,21 @@ Main CLI application with improved command structure. # # Additional attribution and requirements are provided in the NOTICE file. -import sys -from typing import List, Optional import typer from rich.console import Console from rich.traceback import install +from typing import Optional, List +import sys from .commands import ( - ai, - findings, - ingest, - init, - workflow_exec, workflows, -) -from .commands import ( + workflow_exec, + findings, + monitor, config as config_cmd, + ai, + ingest, ) from .constants import DEFAULT_VOLUME_MODE from .fuzzy import enhanced_command_not_found_handler @@ -79,30 +77,25 @@ finding_app = typer.Typer( # === Top-level commands === - @app.command() def init( name: Optional[str] = typer.Option( - None, "--name", "-n", help="Project name (defaults to current directory name)" + None, "--name", "-n", + help="Project name (defaults to current directory name)" ), api_url: Optional[str] = typer.Option( - None, - "--api-url", - "-u", - help="FuzzForge API URL (defaults to http://localhost:8000)", + None, "--api-url", "-u", + help="FuzzForge API URL (defaults to http://localhost:8000)" ), force: bool = typer.Option( - False, - "--force", - "-f", - help="Force initialization even if project already exists", - ), + False, "--force", "-f", + help="Force initialization even if project already exists" + ) ): """ šŸ“ Initialize a new FuzzForge project """ from .commands.init import project - project(name=name, api_url=api_url, force=force) @@ -112,18 +105,39 @@ def status(): šŸ“Š Show project and latest execution status """ from .commands.status import show_status - show_status() +@app.command() +def config( + key: Optional[str] = typer.Argument(None, help="Configuration key"), + value: Optional[str] = typer.Argument(None, help="Configuration value to set") +): + """ + āš™ļø Manage configuration (show all, get, or set values) + """ + + if key is None: + # No arguments: show all config + config_cmd.show_config(global_config=False) + elif value is None: + # Key only: get specific value + config_cmd.get_config(key=key, global_config=False) + else: + # Key and value: set value + config_cmd.set_config(key=key, value=value, global_config=False) + + @app.command() def clean( days: int = typer.Option( - 90, "--days", "-d", help="Remove data older than this many days" + 90, "--days", "-d", + help="Remove data older than this many days" ), dry_run: bool = typer.Option( - False, "--dry-run", help="Show what would be deleted without actually deleting" - ), + False, "--dry-run", + help="Show what would be deleted without actually deleting" + ) ): """ šŸ§¹ Clean old execution data and findings @@ -139,9 +153,7 @@ def clean( raise typer.Exit(1) if dry_run: - console.print( - f"šŸ” [bold]Dry run:[/bold] Would clean data older than {days} days" - ) + console.print(f"šŸ” [bold]Dry run:[/bold] Would clean data older than {days} days") deleted = db.cleanup_old_runs(keep_days=days) @@ -163,38 +175,57 @@ workflow_app.command("retry")(workflow_exec.retry_workflow) workflow_app.command("info")(workflows.workflow_info) workflow_app.command("params")(workflows.workflow_parameters) - @workflow_app.command("run") def run_workflow( workflow: str = typer.Argument(help="Workflow name"), target: str = typer.Argument(help="Target path"), - params: List[str] = typer.Argument( - default=None, help="Parameters as key=value pairs" - ), + params: List[str] = typer.Argument(default=None, help="Parameters as key=value pairs"), param_file: Optional[str] = typer.Option( - None, "--param-file", "-f", help="JSON file containing workflow parameters" + None, "--param-file", "-f", + help="JSON file containing workflow parameters" ), volume_mode: str = typer.Option( - DEFAULT_VOLUME_MODE, - "--volume-mode", - "-v", - help="Volume mount mode: ro (read-only) or rw (read-write)", + DEFAULT_VOLUME_MODE, "--volume-mode", "-v", + help="Volume mount mode: ro (read-only) or rw (read-write)" ), timeout: Optional[int] = typer.Option( - None, "--timeout", "-t", help="Execution timeout in seconds" + None, "--timeout", "-t", + help="Execution timeout in seconds" ), interactive: bool = typer.Option( - True, - "--interactive/--no-interactive", - "-i/-n", - help="Interactive parameter input for missing required parameters", + True, "--interactive/--no-interactive", "-i/-n", + help="Interactive parameter input for missing required parameters" ), wait: bool = typer.Option( - False, "--wait", "-w", help="Wait for execution to complete" + False, "--wait", "-w", + help="Wait for execution to complete" ), + live: bool = typer.Option( + False, "--live", "-l", + help="Start live monitoring after execution (useful for fuzzing workflows)" + ), + auto_start: Optional[bool] = typer.Option( + None, "--auto-start/--no-auto-start", + help="Automatically start required worker if not running (default: from config)" + ), + auto_stop: Optional[bool] = typer.Option( + None, "--auto-stop/--no-auto-stop", + help="Automatically stop worker after execution completes (default: from config)" + ), + fail_on: Optional[str] = typer.Option( + None, "--fail-on", + help="Fail build if findings match SARIF level (error,warning,note,info,all,none). Use with --wait" + ), + export_sarif: Optional[str] = typer.Option( + None, "--export-sarif", + help="Export SARIF results to file after completion. Use with --wait" + ) ): """ šŸš€ Execute a security testing workflow + + Use --fail-on with --wait to fail CI builds based on finding severity. + Use --export-sarif with --wait to export SARIF findings to a file. """ from .commands.workflow_exec import execute_workflow @@ -207,9 +238,13 @@ def run_workflow( timeout=timeout, interactive=interactive, wait=wait, + live=live, + auto_start=auto_start, + auto_stop=auto_stop, + fail_on=fail_on, + export_sarif=export_sarif ) - @workflow_app.callback() def workflow_main(): """ @@ -225,18 +260,17 @@ def workflow_main(): # === Finding commands (singular) === - @finding_app.command("export") def export_finding( - execution_id: Optional[str] = typer.Argument( - None, help="Execution ID (defaults to latest)" - ), + execution_id: Optional[str] = typer.Argument(None, help="Execution ID (defaults to latest)"), format: str = typer.Option( - "sarif", "--format", "-f", help="Export format: sarif, json, csv" + "sarif", "--format", "-f", + help="Export format: sarif, json, csv" ), output: Optional[str] = typer.Option( - None, "--output", "-o", help="Output file (defaults to stdout)" - ), + None, "--output", "-o", + help="Output file (defaults to stdout)" + ) ): """ šŸ“¤ Export findings to file @@ -257,9 +291,7 @@ def export_finding( execution_id = recent_runs[0].run_id console.print(f"šŸ” Using most recent execution: {execution_id}") else: - console.print( - "āš ļø No findings found in project database", style="yellow" - ) + console.print("āš ļø No findings found in project database", style="yellow") return else: console.print("āŒ No project database found", style="red") @@ -272,16 +304,14 @@ def export_finding( @finding_app.command("analyze") def analyze_finding( - finding_id: Optional[str] = typer.Argument(None, help="Finding ID to analyze"), + finding_id: Optional[str] = typer.Argument(None, help="Finding ID to analyze") ): """ šŸ¤– AI analysis of a finding """ from .commands.ai import analyze_finding as ai_analyze - ai_analyze(finding_id) - @finding_app.callback(invoke_without_command=True) def finding_main( ctx: typer.Context, @@ -300,7 +330,7 @@ def finding_main( return # Get remaining arguments for direct viewing - args = ctx.args if hasattr(ctx, "args") else [] + args = ctx.args if hasattr(ctx, 'args') else [] finding_id = args[0] if args else None # Direct viewing: fuzzforge finding [id] @@ -320,9 +350,7 @@ def finding_main( finding_id = recent_runs[0].run_id console.print(f"šŸ” Using most recent execution: {finding_id}") else: - console.print( - "āš ļø No findings found in project database", style="yellow" - ) + console.print("āš ļø No findings found in project database", style="yellow") return else: console.print("āŒ No project database found", style="red") @@ -344,53 +372,17 @@ app.add_typer(workflow_app, name="workflow", help="šŸš€ Execute and manage workf app.add_typer(finding_app, name="finding", help="šŸ” View and analyze findings") # Other command groups +app.add_typer(monitor.app, name="monitor", help="šŸ“Š Real-time monitoring") app.add_typer(ai.app, name="ai", help="šŸ¤– AI integration features") app.add_typer(ingest.app, name="ingest", help="šŸ§  Ingest knowledge into AI") -app.add_typer(config_cmd.app, name="config", help="āš™ļø Manage configuration settings") - # Help and utility commands -@app.command() -def examples(): - """ - šŸ“š Show usage examples - """ - examples_text = """ -[bold cyan]FuzzForge CLI Examples[/bold cyan] - -[bold]Getting Started:[/bold] - ff init # Initialize a project - ff workflows # List available workflows - ff workflow info afl-fuzzing # Get workflow details - -[bold]Execute Workflows:[/bold] - ff workflow afl-fuzzing ./target # Run fuzzing on target - -[bold]Monitor Execution:[/bold] - ff status # Check latest execution - ff workflow status # Same as above - ff workflow history # Show past executions - -[bold]Review Findings:[/bold] - ff findings # List all findings - ff finding # Show latest finding - ff finding export --format sarif # Export findings - -[bold]AI Features:[/bold] - ff ai chat # Interactive AI chat - ff ai suggest ./src # Get workflow suggestions - ff finding analyze # AI analysis of latest finding -""" - console.print(examples_text) - - @app.command() def version(): """ šŸ“¦ Show version information """ from . import __version__ - console.print(f"FuzzForge CLI v{__version__}") console.print("Short command: ff") @@ -399,7 +391,8 @@ def version(): def main_callback( ctx: typer.Context, version: Optional[bool] = typer.Option( - None, "--version", "-v", help="Show version information" + None, "--version", "-v", + help="Show version information" ), ): """ @@ -409,11 +402,9 @@ def main_callback( ā€¢ ff init - Initialize a new project ā€¢ ff workflows - See available workflows ā€¢ ff workflow - Execute a workflow - ā€¢ ff examples - Show usage examples """ if version: from . import __version__ - console.print(f"FuzzForge CLI v{__version__}") raise typer.Exit() @@ -424,11 +415,12 @@ def main(): if len(sys.argv) > 1: args = sys.argv[1:] + # Handle finding command with pattern recognition - if len(args) >= 2 and args[0] == "finding": - finding_subcommands = ["export", "analyze"] + if len(args) >= 2 and args[0] == 'finding': + finding_subcommands = ['export', 'analyze'] # Skip custom dispatching if help flags are present - if not any(arg in ["--help", "-h", "--version", "-v"] for arg in args): + if not any(arg in ['--help', '-h', '--version', '-v'] for arg in args): if args[1] not in finding_subcommands: # Direct finding display: ff finding from .commands.findings import get_findings @@ -448,26 +440,18 @@ def main(): app() except SystemExit as e: # Enhanced error handling for command not found - if hasattr(e, "code") and e.code != 0 and len(sys.argv) > 1: + if hasattr(e, 'code') and e.code != 0 and len(sys.argv) > 1: command_parts = sys.argv[1:] - clean_parts = [part for part in command_parts if not part.startswith("-")] + clean_parts = [part for part in command_parts if not part.startswith('-')] if clean_parts: main_cmd = clean_parts[0] valid_commands = [ - "init", - "status", - "config", - "clean", - "workflows", - "workflow", - "findings", - "finding", - "monitor", - "ai", - "ingest", - "examples", - "version", + 'init', 'status', 'config', 'clean', + 'workflows', 'workflow', + 'findings', 'finding', + 'monitor', 'ai', 'ingest', + 'version' ] if main_cmd not in valid_commands: diff --git a/cli/src/fuzzforge_cli/progress.py b/cli/src/fuzzforge_cli/progress.py index e73b19f..d9f1696 100644 --- a/cli/src/fuzzforge_cli/progress.py +++ b/cli/src/fuzzforge_cli/progress.py @@ -16,10 +16,9 @@ Provides rich progress bars, spinners, and status displays for all long-running import time -import asyncio from contextlib import contextmanager -from typing import Optional, Callable, Any, Dict, List -from datetime import datetime, timedelta +from typing import Optional, Any, Dict, List +from datetime import datetime from rich.console import Console from rich.progress import ( diff --git a/cli/src/fuzzforge_cli/validation.py b/cli/src/fuzzforge_cli/validation.py index 3246fa7..1f524f6 100644 --- a/cli/src/fuzzforge_cli/validation.py +++ b/cli/src/fuzzforge_cli/validation.py @@ -15,7 +15,7 @@ Input validation utilities for FuzzForge CLI. import re from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional from .constants import SUPPORTED_VOLUME_MODES, SUPPORTED_EXPORT_FORMATS from .exceptions import ValidationError diff --git a/cli/src/fuzzforge_cli/worker_manager.py b/cli/src/fuzzforge_cli/worker_manager.py new file mode 100644 index 0000000..2af758b --- /dev/null +++ b/cli/src/fuzzforge_cli/worker_manager.py @@ -0,0 +1,286 @@ +""" +Worker lifecycle management for FuzzForge CLI. + +Manages on-demand startup and shutdown of Temporal workers using Docker Compose. +""" +# Copyright (c) 2025 FuzzingLabs +# +# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file +# at the root of this repository for details. +# +# After the Change Date (four years from publication), this version of the +# Licensed Work will be made available under the Apache License, Version 2.0. +# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 +# +# Additional attribution and requirements are provided in the NOTICE file. + +import logging +import subprocess +import time +from pathlib import Path +from typing import Optional, Dict, Any + +from rich.console import Console + +logger = logging.getLogger(__name__) +console = Console() + + +class WorkerManager: + """ + Manages Temporal worker lifecycle using docker-compose. + + This class handles: + - Checking if workers are running + - Starting workers on demand + - Waiting for workers to be ready + - Stopping workers when done + """ + + def __init__( + self, + compose_file: Optional[Path] = None, + startup_timeout: int = 60, + health_check_interval: float = 2.0 + ): + """ + Initialize WorkerManager. + + Args: + compose_file: Path to docker-compose.yml (defaults to auto-detect) + startup_timeout: Maximum seconds to wait for worker startup + health_check_interval: Seconds between health checks + """ + self.compose_file = compose_file or self._find_compose_file() + self.startup_timeout = startup_timeout + self.health_check_interval = health_check_interval + + def _find_compose_file(self) -> Path: + """ + Auto-detect docker-compose.yml location. + + Searches upward from current directory to find the compose file. + """ + current = Path.cwd() + + # Try current directory and parents + for parent in [current] + list(current.parents): + compose_path = parent / "docker-compose.yml" + if compose_path.exists(): + return compose_path + + # Fallback to default location + return Path("docker-compose.yml") + + def _run_docker_compose(self, *args: str) -> subprocess.CompletedProcess: + """ + Run docker-compose command. + + Args: + *args: Arguments to pass to docker-compose + + Returns: + CompletedProcess with result + + Raises: + subprocess.CalledProcessError: If command fails + """ + cmd = ["docker-compose", "-f", str(self.compose_file)] + list(args) + logger.debug(f"Running: {' '.join(cmd)}") + + return subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + + def is_worker_running(self, container_name: str) -> bool: + """ + Check if a worker container is running. + + Args: + container_name: Name of the Docker container (e.g., "fuzzforge-worker-ossfuzz") + + Returns: + True if container is running, False otherwise + """ + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + check=False + ) + + # Output is "true" or "false" + return result.stdout.strip().lower() == "true" + + except Exception as e: + logger.debug(f"Failed to check worker status: {e}") + return False + + def start_worker(self, container_name: str) -> bool: + """ + Start a worker container using docker. + + Args: + container_name: Name of the Docker container to start + + Returns: + True if started successfully, False otherwise + """ + try: + console.print(f"šŸš€ Starting worker: {container_name}") + + # Use docker start directly (works with container name) + subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + check=True + ) + + logger.info(f"Worker {container_name} started") + return True + + except subprocess.CalledProcessError as e: + logger.error(f"Failed to start worker {container_name}: {e.stderr}") + console.print(f"āŒ Failed to start worker: {e.stderr}", style="red") + return False + + except Exception as e: + logger.error(f"Unexpected error starting worker {container_name}: {e}") + console.print(f"āŒ Unexpected error: {e}", style="red") + return False + + def wait_for_worker_ready(self, container_name: str, timeout: Optional[int] = None) -> bool: + """ + Wait for a worker to be healthy and ready to process tasks. + + Args: + container_name: Name of the Docker container + timeout: Maximum seconds to wait (uses instance default if not specified) + + Returns: + True if worker is ready, False if timeout reached + + Raises: + TimeoutError: If worker doesn't become ready within timeout + """ + timeout = timeout or self.startup_timeout + start_time = time.time() + + console.print("ā³ Waiting for worker to be ready...") + + while time.time() - start_time < timeout: + # Check if container is running + if not self.is_worker_running(container_name): + logger.debug(f"Worker {container_name} not running yet") + time.sleep(self.health_check_interval) + continue + + # Check container health status + try: + result = subprocess.run( + ["docker", "inspect", "-f", "{{.State.Health.Status}}", container_name], + capture_output=True, + text=True, + check=False + ) + + health_status = result.stdout.strip() + + # If no health check is defined, assume healthy after running + if health_status == "" or health_status == "": + logger.info(f"Worker {container_name} is running (no health check)") + console.print(f"āœ… Worker ready: {container_name}") + return True + + if health_status == "healthy": + logger.info(f"Worker {container_name} is healthy") + console.print(f"āœ… Worker ready: {container_name}") + return True + + logger.debug(f"Worker {container_name} health: {health_status}") + + except Exception as e: + logger.debug(f"Failed to check health: {e}") + + time.sleep(self.health_check_interval) + + elapsed = time.time() - start_time + logger.warning(f"Worker {container_name} did not become ready within {elapsed:.1f}s") + console.print(f"āš ļø Worker startup timeout after {elapsed:.1f}s", style="yellow") + return False + + def stop_worker(self, container_name: str) -> bool: + """ + Stop a worker container using docker. + + Args: + container_name: Name of the Docker container to stop + + Returns: + True if stopped successfully, False otherwise + """ + try: + console.print(f"šŸ›‘ Stopping worker: {container_name}") + + # Use docker stop directly (works with container name) + subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + check=True + ) + + logger.info(f"Worker {container_name} stopped") + return True + + except subprocess.CalledProcessError as e: + logger.error(f"Failed to stop worker {container_name}: {e.stderr}") + console.print(f"āŒ Failed to stop worker: {e.stderr}", style="red") + return False + + except Exception as e: + logger.error(f"Unexpected error stopping worker {container_name}: {e}") + console.print(f"āŒ Unexpected error: {e}", style="red") + return False + + def ensure_worker_running( + self, + worker_info: Dict[str, Any], + auto_start: bool = True + ) -> bool: + """ + Ensure a worker is running, starting it if necessary. + + Args: + worker_info: Worker information dict from API (contains worker_container, etc.) + auto_start: Whether to automatically start the worker if not running + + Returns: + True if worker is running, False otherwise + """ + container_name = worker_info["worker_container"] + vertical = worker_info["vertical"] + + # Check if already running + if self.is_worker_running(container_name): + console.print(f"āœ“ Worker already running: {vertical}") + return True + + if not auto_start: + console.print( + f"āš ļø Worker not running: {vertical}. Use --auto-start to start automatically.", + style="yellow" + ) + return False + + # Start the worker + if not self.start_worker(container_name): + return False + + # Wait for it to be ready + return self.wait_for_worker_ready(container_name) diff --git a/cli/uv.lock b/cli/uv.lock index 3d89b0e..841c873 100644 --- a/cli/uv.lock +++ b/cli/uv.lock @@ -1257,7 +1257,7 @@ wheels = [ [[package]] name = "fuzzforge-ai" -version = "0.1.0" +version = "0.6.0" source = { editable = "../ai" } dependencies = [ { name = "a2a-sdk" }, @@ -1303,7 +1303,7 @@ dev = [ [[package]] name = "fuzzforge-cli" -version = "0.1.0" +version = "0.6.0" source = { editable = "." } dependencies = [ { name = "fuzzforge-ai" }, @@ -1347,7 +1347,7 @@ provides-extras = ["dev"] [[package]] name = "fuzzforge-sdk" -version = "0.1.0" +version = "0.6.0" source = { editable = "../sdk" } dependencies = [ { name = "httpx" }, diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..a285b2a --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,110 @@ +# Docker Compose Override for CI/CD Environments +# +# This file optimizes FuzzForge for ephemeral CI/CD environments where: +# - Data persistence is not needed +# - Fast startup is critical +# - Disk I/O can be bypassed +# +# Usage: +# docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d +# +# Benefits: +# - Faster startup (tmpfs instead of volumes) +# - Reduced disk I/O +# - Automatic cleanup (no persistent data) +# +# WARNING: All data is lost when containers stop! + +version: '3.8' + +services: + # Temporal - Use in-memory storage and faster health checks + temporal: + environment: + # Skip some init steps for faster startup + - SKIP_DEFAULT_NAMESPACE_CREATION=false + healthcheck: + # More aggressive health checking for faster feedback + interval: 5s + timeout: 3s + retries: 15 + restart: "no" # Don't restart in CI + + # PostgreSQL - Use in-memory storage and disable durability features + postgresql: + command: > + postgres + -c fsync=off + -c full_page_writes=off + -c synchronous_commit=off + -c wal_level=minimal + -c max_wal_senders=0 + tmpfs: + # Store database in RAM (fast, but ephemeral) + - /var/lib/postgresql/data + healthcheck: + interval: 3s + timeout: 3s + retries: 10 + restart: "no" + + # MinIO - Use in-memory storage + minio: + environment: + # Already set in main compose, but ensure CI mode is enabled + - MINIO_CI_CD=true + tmpfs: + # Store objects in RAM + - /data + healthcheck: + interval: 3s + timeout: 3s + retries: 10 + restart: "no" + + # Backend - Optimize for CI + backend: + environment: + # Add CI-specific environment variables if needed + - CI=true + - LOG_LEVEL=WARNING # Reduce log noise + healthcheck: + interval: 5s + timeout: 3s + retries: 15 + restart: "no" + + # Temporal UI - Disable in CI (not needed, saves resources) + temporal-ui: + profiles: + - ui # Don't start unless explicitly requested + + # MinIO Setup - Speed up bucket creation + minio-setup: + restart: "no" + +# Volumes - Use tmpfs for all persistent data in CI +# Note: This overrides the named volumes with in-memory storage +volumes: + temporal_data: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + + temporal_postgres: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + + minio_data: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + +# Networks - Keep the same +networks: + fuzzforge-network: + driver: bridge diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 5cbe78c..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,234 +0,0 @@ -services: - registry: - image: registry:2 - restart: unless-stopped - ports: - - "5001:5000" - volumes: - - registry_data:/var/lib/registry - healthcheck: - test: ["CMD-SHELL", "wget -q --spider http://localhost:5000/v2/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - - postgres: - image: postgres:14 - environment: - POSTGRES_USER: prefect - POSTGRES_PASSWORD: prefect - POSTGRES_DB: prefect - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U prefect"] - interval: 5s - timeout: 5s - retries: 5 - - redis: - image: redis:7 - volumes: - - redis_data:/data - healthcheck: - test: ["CMD-SHELL", "redis-cli ping"] - interval: 5s - timeout: 5s - retries: 5 - - prefect-server: - image: prefecthq/prefect:3-latest - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - environment: - PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://prefect:prefect@postgres:5432/prefect - PREFECT_SERVER_API_HOST: 0.0.0.0 - PREFECT_API_URL: http://prefect-server:4200/api - PREFECT_MESSAGING_BROKER: prefect_redis.messaging - PREFECT_MESSAGING_CACHE: prefect_redis.messaging - PREFECT_REDIS_MESSAGING_HOST: redis - PREFECT_REDIS_MESSAGING_PORT: 6379 - PREFECT_REDIS_MESSAGING_DB: 0 - PREFECT_LOCAL_STORAGE_PATH: /prefect-storage - PREFECT_RESULTS_PERSIST_BY_DEFAULT: "true" - command: > - sh -c " - mkdir -p /prefect-storage && - chmod 755 /prefect-storage && - prefect server start --no-services - " - ports: - - "4200:4200" - volumes: - - prefect_storage:/prefect-storage - - prefect-services: - image: prefecthq/prefect:3-latest - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - environment: - PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://prefect:prefect@postgres:5432/prefect - PREFECT_MESSAGING_BROKER: prefect_redis.messaging - PREFECT_MESSAGING_CACHE: prefect_redis.messaging - PREFECT_REDIS_MESSAGING_HOST: redis - PREFECT_REDIS_MESSAGING_PORT: 6379 - PREFECT_REDIS_MESSAGING_DB: 0 - PREFECT_LOCAL_STORAGE_PATH: /prefect-storage - PREFECT_RESULTS_PERSIST_BY_DEFAULT: "true" - command: > - sh -c " - mkdir -p /prefect-storage && - chmod 755 /prefect-storage && - prefect server services start - " - volumes: - - prefect_storage:/prefect-storage - - docker-proxy: - image: tecnativa/docker-socket-proxy - environment: - # Enable permissions needed for Prefect worker container creation and management - CONTAINERS: 1 - IMAGES: 1 - BUILD: 1 - VOLUMES: 1 - NETWORKS: 1 - SERVICES: 1 # Required for some container operations - TASKS: 1 # Required for container management - NODES: 1 # Required for container scheduling - GET: 1 - POST: 1 - PUT: 1 - DELETE: 1 - HEAD: 1 - INFO: 1 - VERSION: 1 - PING: 1 - EVENTS: 1 - DISTRIBUTION: 1 - AUTH: 1 - # Still block the most dangerous operations - SYSTEM: 0 - SWARM: 0 - EXEC: 0 # Keep container exec blocked for security - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - ports: - - "2375" - networks: - - default - - prefect-worker: - image: prefecthq/prefect:3-latest - depends_on: - prefect-server: - condition: service_started - docker-proxy: - condition: service_started - registry: - condition: service_healthy - environment: - PREFECT_API_URL: http://prefect-server:4200/api - PREFECT_LOCAL_STORAGE_PATH: /prefect-storage - PREFECT_RESULTS_PERSIST_BY_DEFAULT: "true" - DOCKER_HOST: tcp://docker-proxy:2375 - DOCKER_BUILDKIT: 1 # Enable BuildKit for better performance - DOCKER_CONFIG: /tmp/docker - # Registry URLs (set REGISTRY_HOST in your environment or .env) - # - macOS/Windows Docker Desktop: REGISTRY_HOST=host.docker.internal - # - Linux: REGISTRY_HOST=localhost (default) - FUZZFORGE_REGISTRY_PUSH_URL: "${REGISTRY_HOST:-localhost}:5001" - FUZZFORGE_REGISTRY_PULL_URL: "${REGISTRY_HOST:-localhost}:5001" - command: > - sh -c " - mkdir -p /tmp/docker && - mkdir -p /prefect-storage && - chmod 755 /prefect-storage && - echo '{\"insecure-registries\": [\"registry:5000\", \"localhost:5001\", \"host.docker.internal:5001\"]}' > /tmp/docker/config.json && - pip install 'prefect[docker]' && - echo 'Waiting for backend to create work pool...' && - sleep 15 && - prefect worker start --pool docker-pool --type docker - " - volumes: - - prefect_storage:/prefect-storage # Access to shared storage for results - - toolbox_code:/opt/prefect/toolbox:ro # Access to toolbox code for building - networks: - - default - extra_hosts: - - "host.docker.internal:host-gateway" - - fuzzforge-backend: - build: - context: ./backend - dockerfile: Dockerfile - depends_on: - prefect-server: - condition: service_started - docker-proxy: - condition: service_started - registry: - condition: service_healthy - environment: - PREFECT_API_URL: http://prefect-server:4200/api - PREFECT_LOCAL_STORAGE_PATH: /prefect-storage - PREFECT_RESULTS_PERSIST_BY_DEFAULT: "true" - DOCKER_HOST: tcp://docker-proxy:2375 - DOCKER_BUILDKIT: 1 - DOCKER_CONFIG: /tmp/docker - DOCKER_TLS_VERIFY: "" - DOCKER_REGISTRY_INSECURE: "registry:5000,localhost:5001,host.docker.internal:5001" - # Registry URLs (set REGISTRY_HOST in your environment or .env) - # - macOS/Windows Docker Desktop: REGISTRY_HOST=host.docker.internal - # - Linux: REGISTRY_HOST=localhost (default) - FUZZFORGE_REGISTRY_PUSH_URL: "${REGISTRY_HOST:-localhost}:5001" - FUZZFORGE_REGISTRY_PULL_URL: "${REGISTRY_HOST:-localhost}:5001" - ports: - - "8000:8000" - - "8010:8010" - volumes: - - prefect_storage:/prefect-storage - - ./backend/toolbox:/app/toolbox:ro # Direct host mount (read-only) for live updates - - toolbox_code:/opt/prefect/toolbox # Share toolbox code with workers - - ./test_projects:/app/test_projects:ro # Test projects for workflow testing - networks: - - default - extra_hosts: - - "host.docker.internal:host-gateway" - # Sync toolbox code to shared volume and start server with live reload - command: > - sh -c " - mkdir -p /opt/prefect/toolbox && - mkdir -p /prefect-storage && - mkdir -p /tmp/docker && - chmod 755 /prefect-storage && - echo '{\"insecure-registries\": [\"registry:5000\", \"localhost:5001\", \"host.docker.internal:5001\"]}' > /tmp/docker/config.json && - cp -r /app/toolbox/* /opt/prefect/toolbox/ 2>/dev/null || true && - (while true; do - rsync -av --delete /app/toolbox/ /opt/prefect/toolbox/ > /dev/null 2>&1 || true - sleep 10 - done) & - uv run uvicorn src.main:app --host 0.0.0.0 --port 8000 --reload - " - -volumes: - postgres_data: - name: fuzzforge_postgres_data - redis_data: - name: fuzzforge_redis_data - prefect_storage: - name: fuzzforge_prefect_storage - toolbox_code: - name: fuzzforge_toolbox_code - registry_data: - name: fuzzforge_registry_data - -networks: - default: - name: fuzzforge_default diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f55e5ec --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,609 @@ +# FuzzForge AI - Temporal Architecture with Vertical Workers +# +# This is the new architecture using: +# - Temporal for workflow orchestration +# - MinIO for unified storage (dev + prod) +# - Vertical workers with pre-built toolchains +# +# Usage: +# Development: docker-compose -f docker-compose.temporal.yaml up +# Production: docker-compose -f docker-compose.temporal.yaml -f docker-compose.temporal.prod.yaml up + +version: '3.8' + +services: + # ============================================================================ + # Temporal Server - Workflow Orchestration + # ============================================================================ + temporal: + image: temporalio/auto-setup:latest + container_name: fuzzforge-temporal + depends_on: + - postgresql + ports: + - "7233:7233" # gRPC API + environment: + # Database configuration + - DB=postgres12 + - DB_PORT=5432 + - POSTGRES_USER=temporal + - POSTGRES_PWD=temporal + - POSTGRES_SEEDS=postgresql + # Temporal configuration (no custom dynamic config) + - ENABLE_ES=false + - ES_SEEDS= + # Address configuration + - TEMPORAL_ADDRESS=temporal:7233 + - TEMPORAL_CLI_ADDRESS=temporal:7233 + volumes: + - temporal_data:/etc/temporal + networks: + - fuzzforge-network + healthcheck: + test: ["CMD", "tctl", "--address", "temporal:7233", "cluster", "health"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + # ============================================================================ + # Temporal UI - Web Interface + # ============================================================================ + temporal-ui: + image: temporalio/ui:latest + container_name: fuzzforge-temporal-ui + depends_on: + - temporal + ports: + - "8080:8080" # Web UI (http://localhost:8080) + environment: + - TEMPORAL_ADDRESS=temporal:7233 + - TEMPORAL_CORS_ORIGINS=http://localhost:8080 + networks: + - fuzzforge-network + restart: unless-stopped + + # ============================================================================ + # Temporal Database - PostgreSQL (lightweight for dev) + # ============================================================================ + postgresql: + image: postgres:14-alpine + container_name: fuzzforge-temporal-postgresql + environment: + POSTGRES_USER: temporal + POSTGRES_PASSWORD: temporal + POSTGRES_DB: temporal + volumes: + - temporal_postgres:/var/lib/postgresql/data + networks: + - fuzzforge-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U temporal"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # ============================================================================ + # MinIO - S3-Compatible Object Storage + # ============================================================================ + minio: + image: minio/minio:latest + container_name: fuzzforge-minio + command: server /data --console-address ":9001" + ports: + - "9000:9000" # S3 API + - "9001:9001" # Web Console (http://localhost:9001) + environment: + MINIO_ROOT_USER: fuzzforge + MINIO_ROOT_PASSWORD: fuzzforge123 + # Lightweight mode for development (reduces memory to 256MB) + MINIO_CI_CD: "true" + volumes: + - minio_data:/data + networks: + - fuzzforge-network + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # ============================================================================ + # MinIO Setup - Create Buckets and Lifecycle Policies + # ============================================================================ + minio-setup: + image: minio/mc:latest + container_name: fuzzforge-minio-setup + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + echo 'Waiting for MinIO to be ready...'; + sleep 2; + + echo 'Setting up MinIO alias...'; + mc alias set fuzzforge http://minio:9000 fuzzforge fuzzforge123; + + echo 'Creating buckets...'; + mc mb fuzzforge/targets --ignore-existing; + mc mb fuzzforge/results --ignore-existing; + mc mb fuzzforge/cache --ignore-existing; + + echo 'Setting lifecycle policies...'; + mc ilm add fuzzforge/targets --expiry-days 7; + mc ilm add fuzzforge/results --expiry-days 30; + mc ilm add fuzzforge/cache --expiry-days 3; + + echo 'Setting access policies...'; + mc anonymous set download fuzzforge/results; + + echo 'MinIO setup complete!'; + exit 0; + " + networks: + - fuzzforge-network + + # ============================================================================ + # Vertical Worker: Rust/Native Security + # ============================================================================ + # This is a template/example worker. In production, you'll have multiple + # vertical workers (android, rust, web, ios, blockchain, etc.) + worker-rust: + build: + context: ./workers/rust + dockerfile: Dockerfile + container_name: fuzzforge-worker-rust + profiles: + - workers + - rust + depends_on: + postgresql: + condition: service_healthy + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + # Temporal configuration + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_NAMESPACE: default + + # Worker configuration + WORKER_VERTICAL: rust + WORKER_TASK_QUEUE: rust-queue + MAX_CONCURRENT_ACTIVITIES: 5 + + # Storage configuration (MinIO) + STORAGE_BACKEND: s3 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + S3_REGION: us-east-1 + S3_USE_SSL: "false" + + # Cache configuration + CACHE_DIR: /cache + CACHE_MAX_SIZE: 10GB + CACHE_TTL: 7d + + # Logging + LOG_LEVEL: INFO + PYTHONUNBUFFERED: 1 + volumes: + # Mount workflow code (read-only) for dynamic discovery + - ./backend/toolbox:/app/toolbox:ro + # Worker cache for downloaded targets + - worker_rust_cache:/cache + networks: + - fuzzforge-network + restart: "no" + # Resource limits (adjust based on vertical needs) + deploy: + resources: + limits: + cpus: '2' + memory: 2G + reservations: + cpus: '1' + memory: 512M + + # ============================================================================ + # Vertical Worker: Python Fuzzing + # ============================================================================ + worker-python: + build: + context: ./workers/python + dockerfile: Dockerfile + container_name: fuzzforge-worker-python + profiles: + - workers + - python + depends_on: + postgresql: + condition: service_healthy + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + # Temporal configuration + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_NAMESPACE: default + + # Worker configuration + WORKER_VERTICAL: python + WORKER_TASK_QUEUE: python-queue + MAX_CONCURRENT_ACTIVITIES: 5 + + # Storage configuration (MinIO) + STORAGE_BACKEND: s3 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + S3_REGION: us-east-1 + S3_USE_SSL: "false" + + # Cache configuration + CACHE_DIR: /cache + CACHE_MAX_SIZE: 10GB + CACHE_TTL: 7d + + # Logging + LOG_LEVEL: INFO + PYTHONUNBUFFERED: 1 + volumes: + # Mount workflow code (read-only) for dynamic discovery + - ./backend/toolbox:/app/toolbox:ro + # Mount AI module for A2A wrapper access + - ./ai/src:/app/ai_src:ro + # Worker cache for downloaded targets + - worker_python_cache:/cache + networks: + - fuzzforge-network + restart: "no" + # Resource limits (lighter than rust) + deploy: + resources: + limits: + cpus: '1' + memory: 1G + reservations: + cpus: '0.5' + memory: 256M + + # ============================================================================ + # Vertical Worker: Secret Detection + # ============================================================================ + worker-secrets: + build: + context: ./workers/secrets + dockerfile: Dockerfile + container_name: fuzzforge-worker-secrets + profiles: + - workers + - secrets + depends_on: + postgresql: + condition: service_healthy + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + # Temporal configuration + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_NAMESPACE: default + + # Worker configuration + WORKER_VERTICAL: secrets + WORKER_TASK_QUEUE: secrets-queue + MAX_CONCURRENT_ACTIVITIES: 5 + + # Storage configuration (MinIO) + STORAGE_BACKEND: s3 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + S3_REGION: us-east-1 + S3_USE_SSL: "false" + + # Cache configuration + CACHE_DIR: /cache + CACHE_MAX_SIZE: 10GB + CACHE_TTL: 7d + + # Logging + LOG_LEVEL: INFO + PYTHONUNBUFFERED: 1 + volumes: + # Mount workflow code (read-only) for dynamic discovery + - ./backend/toolbox:/app/toolbox:ro + # Mount AI module for A2A wrapper access + - ./ai/src:/app/ai_src:ro + # Worker cache for downloaded targets + - worker_secrets_cache:/cache + networks: + - fuzzforge-network + restart: "no" + # Resource limits (lighter than rust) + deploy: + resources: + limits: + cpus: '1' + memory: 1G + reservations: + cpus: '0.5' + memory: 256M + + # ============================================================================ + # Vertical Worker: Android Security + # ============================================================================ + worker-android: + build: + context: ./workers/android + dockerfile: Dockerfile + container_name: fuzzforge-worker-android + profiles: + - workers + - android + - full + depends_on: + postgresql: + condition: service_healthy + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + # Temporal configuration + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_NAMESPACE: default + + # Worker configuration + WORKER_VERTICAL: android + WORKER_TASK_QUEUE: android-queue + MAX_CONCURRENT_ACTIVITIES: 5 + + # Storage configuration (MinIO) + STORAGE_BACKEND: s3 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + S3_REGION: us-east-1 + S3_USE_SSL: "false" + + # Cache configuration + CACHE_DIR: /cache + CACHE_MAX_SIZE: 10GB + CACHE_TTL: 7d + + # Logging + LOG_LEVEL: INFO + PYTHONUNBUFFERED: 1 + volumes: + # Mount workflow code (read-only) for dynamic discovery + - ./backend/toolbox:/app/toolbox:ro + # Worker cache for downloaded targets + - worker_android_cache:/cache + networks: + - fuzzforge-network + restart: "no" + # Resource limits (Android tools need more memory) + deploy: + resources: + limits: + cpus: '2' + memory: 3G + reservations: + cpus: '1' + memory: 1G + + # ============================================================================ + # FuzzForge Backend API + # ============================================================================ + backend: + build: + context: ./backend + dockerfile: Dockerfile + container_name: fuzzforge-backend + depends_on: + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + # Temporal configuration + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_NAMESPACE: default + + # Storage configuration (MinIO) + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + S3_REGION: us-east-1 + S3_USE_SSL: "false" + + # Python configuration + PYTHONPATH: /app + PYTHONUNBUFFERED: 1 + + # Logging + LOG_LEVEL: INFO + ports: + - "8000:8000" # FastAPI REST API + - "8010:8010" # MCP (Model Context Protocol) + volumes: + # Mount toolbox for workflow discovery (read-only) + - ./backend/toolbox:/app/toolbox:ro + networks: + - fuzzforge-network + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + + # ============================================================================ + # Task Agent - A2A LiteLLM Agent + # ============================================================================ + task-agent: + build: + context: ./ai/agents/task_agent + dockerfile: Dockerfile + container_name: fuzzforge-task-agent + ports: + - "10900:8000" + env_file: + - ./volumes/env/.env + environment: + - PORT=8000 + - PYTHONUNBUFFERED=1 + volumes: + - ./volumes/env:/app/config:ro + networks: + - fuzzforge-network + restart: unless-stopped + + # ============================================================================ + # Vertical Worker: OSS-Fuzz Campaigns + # ============================================================================ + worker-ossfuzz: + build: + context: ./workers/ossfuzz + dockerfile: Dockerfile + container_name: fuzzforge-worker-ossfuzz + profiles: + - workers + - ossfuzz + depends_on: + postgresql: + condition: service_healthy + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + # Temporal configuration + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_NAMESPACE: default + + # Worker configuration + WORKER_VERTICAL: ossfuzz + WORKER_TASK_QUEUE: ossfuzz-queue + MAX_CONCURRENT_ACTIVITIES: 2 # Lower concurrency for resource-intensive fuzzing + + # Storage configuration (MinIO) + STORAGE_BACKEND: s3 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + S3_REGION: us-east-1 + S3_USE_SSL: "false" + + # Cache configuration (larger for OSS-Fuzz builds) + CACHE_DIR: /cache + CACHE_MAX_SIZE: 50GB + CACHE_TTL: 30d + + # Logging + LOG_LEVEL: INFO + PYTHONUNBUFFERED: 1 + volumes: + # Mount workflow code (read-only) for dynamic discovery + - ./backend/toolbox:/app/toolbox:ro + # Worker cache for OSS-Fuzz builds and corpus + - worker_ossfuzz_cache:/cache + # OSS-Fuzz build output + - worker_ossfuzz_build:/opt/oss-fuzz/build + networks: + - fuzzforge-network + restart: "no" + # Higher resource limits for fuzzing campaigns + deploy: + resources: + limits: + cpus: '4' + memory: 8G + reservations: + cpus: '2' + memory: 2G + +# ============================================================================ +# Volumes +# ============================================================================ +volumes: + temporal_data: + name: fuzzforge_temporal_data + temporal_postgres: + name: fuzzforge_temporal_postgres + minio_data: + name: fuzzforge_minio_data + worker_rust_cache: + name: fuzzforge_worker_rust_cache + worker_python_cache: + name: fuzzforge_worker_python_cache + worker_secrets_cache: + name: fuzzforge_worker_secrets_cache + worker_android_cache: + name: fuzzforge_worker_android_cache + worker_ossfuzz_cache: + name: fuzzforge_worker_ossfuzz_cache + worker_ossfuzz_build: + name: fuzzforge_worker_ossfuzz_build + # Add more worker caches as you add verticals: + # worker_web_cache: + # worker_ios_cache: + +# ============================================================================ +# Networks +# ============================================================================ +networks: + fuzzforge-network: + name: fuzzforge_temporal_network + driver: bridge + +# ============================================================================ +# Notes: +# ============================================================================ +# +# 1. First Startup: +# - Creates all buckets and policies automatically +# - Temporal auto-setup creates database schema +# - Takes ~30-60 seconds for all health checks +# +# 2. Adding Vertical Workers: +# - Copy worker-rust section +# - Update: container_name, build.context, WORKER_VERTICAL, volumes +# - Add corresponding cache volume +# +# 3. Scaling Workers: +# - Horizontal: docker-compose up -d --scale worker-rust=3 +# - Vertical: Increase MAX_CONCURRENT_ACTIVITIES env var +# +# 4. Web UIs: +# - Temporal UI: http://localhost:8233 +# - MinIO Console: http://localhost:9001 (user: fuzzforge, pass: fuzzforge123) +# +# 5. Resource Usage (Baseline): +# - Temporal: ~500MB +# - Temporal DB: ~100MB +# - MinIO: ~256MB (with CI_CD=true) +# - Worker-rust: ~512MB (varies by toolchain) +# - Total: ~1.4GB baseline +# +# 6. Production Overrides: +# - Use docker-compose.temporal.prod.yaml for: +# - Disable CI_CD mode (more memory but better performance) +# - Add more workers +# - Increase resource limits +# - Add monitoring/logging diff --git a/docs/docs/ai/a2a-services.md b/docs/docs/ai/a2a-services.md index aac4c94..694be54 100644 --- a/docs/docs/ai/a2a-services.md +++ b/docs/docs/ai/a2a-services.md @@ -68,7 +68,7 @@ Response excerpt: - Call `POST /graph/query` to explore project knowledge. - Call `POST /project/files` to fetch raw files from the repository. - Download finished scan summaries with `GET /artifacts/{id}`. -4. The AI module pushes Prefect workflow results into artifacts automatically, so remote agents can poll without re-running scans. +4. The AI module pushes Temporal workflow results into artifacts automatically, so remote agents can poll without re-running scans. ## Registration Flow @@ -129,7 +129,7 @@ sequenceDiagram participant Remote as Remote Agent participant HTTP as A2A Server participant Exec as Executor - participant Workflow as Prefect Backend + participant Workflow as Temporal Backend Remote->>HTTP: POST / (message with tool request) HTTP->>Exec: Forward message diff --git a/docs/docs/ai/architecture.md b/docs/docs/ai/architecture.md index 60f334b..eea821b 100644 --- a/docs/docs/ai/architecture.md +++ b/docs/docs/ai/architecture.md @@ -1,6 +1,6 @@ # AI Architecture -FuzzForge AI is the orchestration layer that lets large language models drive the broader security platform. Built on the Google ADK runtime, the module coordinates local tools, remote Agent-to-Agent (A2A) peers, and Prefect-backed workflows while persisting long-running context for every project. +FuzzForge AI is the orchestration layer that lets large language models drive the broader security platform. Built on the Google ADK runtime, the module coordinates local tools, remote Agent-to-Agent (A2A) peers, and Temporal-backed workflows while persisting long-running context for every project. ## System Diagram @@ -27,7 +27,7 @@ graph TB Executor --> Prompts[Prompt Templates] Router --> RemoteAgents[Registered A2A Agents] - MCP --> Prefect[FuzzForge Backend] + MCP --> Temporal[FuzzForge Backend] Memory --> SessionDB[Session Store] Memory --> Semantic[Semantic Recall] Memory --> Graphs[Cognee Graph] @@ -44,7 +44,7 @@ sequenceDiagram participant CLI as CLI / HTTP Surface participant Exec as FuzzForgeExecutor participant ADK as ADK Runner - participant Prefect as Prefect Backend + participant Temporal as Temporal Backend participant Cognee as Cognee participant Artifact as Artifact Cache @@ -52,8 +52,8 @@ sequenceDiagram CLI->>Exec: Normalised request + context ID Exec->>ADK: Tool invocation (LiteLLM) ADK-->>Exec: Structured response / tool result - Exec->>Prefect: (optional) submit workflow via MCP - Prefect-->>Exec: Run status updates + Exec->>Temporal: (optional) submit workflow via MCP + Temporal-->>Exec: Run status updates Exec->>Cognee: (optional) knowledge query / ingestion Cognee-->>Exec: Graph results Exec->>Artifact: Persist generated files @@ -69,7 +69,7 @@ sequenceDiagram ## Core Components - **FuzzForgeAgent** (`ai/src/fuzzforge_ai/agent.py`) assembles the runtime: it loads environment variables, constructs the executor, and builds an ADK `Agent` backed by `LiteLlm`. The singleton accessor `get_fuzzforge_agent()` keeps CLI and server instances aligned and shares the generated agent card. -- **FuzzForgeExecutor** (`ai/src/fuzzforge_ai/agent_executor.py`) is the brain. It registers tools, manages session storage (SQLite or in-memory via `DatabaseSessionService` / `InMemorySessionService`), and coordinates artifact storage. The executor also tracks long-running Prefect workflows inside `pending_runs`, produces `TaskStatusUpdateEvent` objects, and funnels every response through ADKā€™s `Runner` so traces include tool metadata. +- **FuzzForgeExecutor** (`ai/src/fuzzforge_ai/agent_executor.py`) is the brain. It registers tools, manages session storage (SQLite or in-memory via `DatabaseSessionService` / `InMemorySessionService`), and coordinates artifact storage. The executor also tracks long-running Temporal workflows inside `pending_runs`, produces `TaskStatusUpdateEvent` objects, and funnels every response through ADKā€™s `Runner` so traces include tool metadata. - **Remote agent registry** (`ai/src/fuzzforge_ai/remote_agent.py`) holds metadata for downstream agents and handles capability discovery over HTTP. Auto-registration is configured by `ConfigManager` so known agents attach on startup. - **Memory services**: - `FuzzForgeMemoryService` and `HybridMemoryManager` (`ai/src/fuzzforge_ai/memory_service.py`) provide conversation recall and bridge to Cognee datasets when configured. @@ -77,15 +77,15 @@ sequenceDiagram ## Workflow Automation -The executor wraps Prefect MCP actions exposed by the backend: +The executor wraps Temporal MCP actions exposed by the backend: | Tool | Source | Purpose | | --- | --- | --- | | `list_workflows_mcp` | `ai/src/fuzzforge_ai/agent_executor.py` | Enumerate available scans | | `submit_security_scan_mcp` | `agent_executor.py` | Launch a scan and persist run metadata | -| `get_run_status_mcp` | `agent_executor.py` | Poll Prefect for status and push task events | +| `get_run_status_mcp` | `agent_executor.py` | Poll Temporal for status and push task events | | `get_comprehensive_scan_summary` | `agent_executor.py` | Collect findings and bundle artifacts | -| `get_backend_status_mcp` | `agent_executor.py` | Block submissions until Prefect reports `ready` | +| `get_backend_status_mcp` | `agent_executor.py` | Block submissions until Temporal reports `ready` | The CLI surface mirrors these helpers as natural-language prompts (`You> run fuzzforge workflow ā€¦`). ADKā€™s `Runner` handles retries and ensures each tool call yields structured `Event` objects for downstream instrumentation. diff --git a/docs/docs/ai/configuration.md b/docs/docs/ai/configuration.md index cb42783..2da0c11 100644 --- a/docs/docs/ai/configuration.md +++ b/docs/docs/ai/configuration.md @@ -87,7 +87,7 @@ If the Cognee variables are omitted, graph-specific tools remain available but r FUZZFORGE_MCP_URL=http://localhost:8010/mcp ``` -The agent uses this endpoint to list, launch, and monitor Prefect workflows. +The agent uses this endpoint to list, launch, and monitor Temporal workflows. ## Tracing & Observability diff --git a/docs/docs/ai/ingestion.md b/docs/docs/ai/ingestion.md index 0af3c9e..8e7ad58 100644 --- a/docs/docs/ai/ingestion.md +++ b/docs/docs/ai/ingestion.md @@ -53,7 +53,7 @@ All runs automatically skip `.fuzzforge/**` and `.git/**` to avoid recursive ing You> refresh the project knowledge graph for ./backend Assistant> Kicks off `fuzzforge ingest` with recursive scan -You> search project knowledge for "prefect workflow" using INSIGHTS +You> search project knowledge for "temporal workflow" using INSIGHTS Assistant> Routes to Cognee `search_project_knowledge` You> ingest_to_dataset("Design doc for new scanner", "insights") @@ -70,7 +70,7 @@ LLM_PROVIDER=openai LITELLM_MODEL=gpt-5-mini OPENAI_API_KEY=sk-your-key -# FuzzForge backend (Prefect-powered) +# FuzzForge backend (Temporal-powered) FUZZFORGE_MCP_URL=http://localhost:8010/mcp # Optional: knowledge graph provider diff --git a/docs/docs/ai/intro.md b/docs/docs/ai/intro.md index 073c4b1..491e200 100644 --- a/docs/docs/ai/intro.md +++ b/docs/docs/ai/intro.md @@ -4,7 +4,7 @@ sidebar_position: 1 # FuzzForge AI Module -FuzzForge AI is the multi-agent layer that lets you operate the FuzzForge security platform through natural language. It orchestrates local tooling, registered Agent-to-Agent (A2A) peers, and the Prefect-powered backend while keeping long-running context in memory and project knowledge graphs. +FuzzForge AI is the multi-agent layer that lets you operate the FuzzForge security platform through natural language. It orchestrates local tooling, registered Agent-to-Agent (A2A) peers, and the Temporal-powered backend while keeping long-running context in memory and project knowledge graphs. ## Quick Start @@ -36,7 +36,7 @@ FuzzForge AI is the multi-agent layer that lets you operate the FuzzForge securi ```bash fuzzforge ai agent ``` - Keep the backend running (Prefect API at `FUZZFORGE_MCP_URL`) so workflow commands succeed. + Keep the backend running (Temporal API at `FUZZFORGE_MCP_URL`) so workflow commands succeed. ## Everyday Workflow @@ -65,7 +65,7 @@ Inside `fuzzforge ai agent` you can mix slash commands and free-form prompts: /sendfile SecurityAgent src/report.md "Please review" You> route_to SecurityAnalyzer: scan ./backend for secrets You> run fuzzforge workflow static_analysis_scan on ./test_projects/demo -You> search project knowledge for "prefect status" using INSIGHTS +You> search project knowledge for "temporal status" using INSIGHTS ``` Artifacts created during the conversation are served from `.fuzzforge/artifacts/` and exposed through the A2A HTTP API. @@ -88,7 +88,7 @@ Use these to validate the setup once the agent shell is running: - `run fuzzforge workflow static_analysis_scan on ./backend with target_branch=main` - `show findings for that run once it finishes` - `refresh the project knowledge graph for ./backend` -- `search project knowledge for "prefect readiness" using INSIGHTS` +- `search project knowledge for "temporal readiness" using INSIGHTS` - `/recall terraform secrets` - `/memory status` - `ROUTE_TO SecurityAnalyzer: audit infrastructure_vulnerable` diff --git a/docs/docs/ai/prompts.md b/docs/docs/ai/prompts.md index 8649b7f..7ac5859 100644 --- a/docs/docs/ai/prompts.md +++ b/docs/docs/ai/prompts.md @@ -33,7 +33,7 @@ Assistant> Streams the `get_comprehensive_scan_summary` output and attaches the You> refresh the project knowledge graph for ./backend Assistant> Launches `fuzzforge ingest --path ./backend --recursive` and reports file counts. -You> search project knowledge for "prefect readiness" using INSIGHTS +You> search project knowledge for "temporal readiness" using INSIGHTS Assistant> Routes to Cognee via `query_project_knowledge_api` and returns the top matches. You> recall "api key rotation" @@ -52,7 +52,7 @@ Assistant> Uploads the file as an artifact and notifies the remote agent. ## Prompt Tips -- Use explicit verbs (`list`, `run`, `search`) to trigger the Prefect workflow helpers. +- Use explicit verbs (`list`, `run`, `search`) to trigger the Temporal workflow helpers. - Include parameter names inline (`with target_branch=main`) so the executor maps values to MCP tool inputs without additional clarification. - When referencing prior runs, reuse the assistantā€™s run IDs or ask for "the last run"ā€”the session store tracks them per context ID. - If Cognee is not configured, graph queries return a friendly notice; set `LLM_COGNEE_*` variables to enable full answers. diff --git a/docs/docs/concept/architecture.md b/docs/docs/concept/architecture.md index f6703b0..a7e7429 100644 --- a/docs/docs/concept/architecture.md +++ b/docs/docs/concept/architecture.md @@ -25,9 +25,9 @@ At a glance, FuzzForge is organized into several layers, each with a clear respo - **Client Layer:** Where users and external systems interact (CLI, API clients, MCP server). - **API Layer:** The FastAPI backend, which exposes REST endpoints and manages requests. -- **Orchestration Layer:** Prefect server and workers, which schedule and execute workflows. -- **Execution Layer:** Docker Engine and containers, where workflows actually run. -- **Storage Layer:** PostgreSQL database, Docker volumes, and a result cache for persistence. +- **Orchestration Layer:** Temporal server and vertical workers, which schedule and execute workflows. +- **Execution Layer:** Long-lived vertical worker containers with pre-installed toolchains, where workflows run. +- **Storage Layer:** PostgreSQL database, MinIO (S3-compatible storage), and worker cache for persistence. Hereā€™s a simplified view of how these layers fit together: @@ -46,8 +46,8 @@ graph TB end subgraph "Orchestration Layer" - Prefect[Prefect Server] - Workers[Prefect Workers] + Temporal[Temporal Server] + Workers[Vertical Workers] Scheduler[Workflow Scheduler] end @@ -69,9 +69,9 @@ graph TB FastAPI --> Router Router --> Middleware - Middleware --> Prefect + Middleware --> Temporal - Prefect --> Workers + Temporal --> Workers Workers --> Scheduler Scheduler --> Docker @@ -93,51 +93,61 @@ graph TB ### Orchestration Layer -- **Prefect Server:** Schedules and tracks workflows, backed by PostgreSQL. -- **Prefect Workers:** Execute workflows in Docker containers. Can be scaled horizontally. -- **Workflow Scheduler:** Balances load, manages priorities, and enforces resource limits. +- **Temporal Server:** Schedules and tracks workflows, backed by PostgreSQL. +- **Vertical Workers:** Long-lived workers pre-built with domain-specific toolchains (Android, Rust, Web, etc.). Can be scaled horizontally. +- **Task Queues:** Route workflows to appropriate vertical workers based on workflow metadata. ### Execution Layer -- **Docker Engine:** Runs workflow containers, enforcing isolation and resource limits. -- **Workflow Containers:** Custom images with security tools, mounting code and results volumes. -- **Docker Registry:** Stores and distributes workflow images. +- **Vertical Workers:** Long-lived processes with pre-installed security tools for specific domains. +- **MinIO Storage:** S3-compatible storage for uploaded targets and results. +- **Worker Cache:** Local cache for downloaded targets, with LRU eviction. ### Storage Layer -- **PostgreSQL Database:** Stores workflow metadata, state, and results. -- **Docker Volumes:** Persist workflow results and artifacts. -- **Result Cache:** Speeds up access to recent results, with in-memory and disk persistence. +- **PostgreSQL Database:** Stores Temporal workflow state and metadata. +- **MinIO (S3):** Persistent storage for uploaded targets and workflow results. +- **Worker Cache:** Local filesystem cache for downloaded targets with workspace isolation: + - **Isolated mode**: Each run gets `/cache/{target_id}/{run_id}/workspace/` + - **Shared mode**: All runs share `/cache/{target_id}/workspace/` + - **Copy-on-write mode**: Download once, copy per run + - **LRU eviction** when cache exceeds configured size ## How Does Data Flow Through the System? ### Submitting a Workflow -1. **User submits a workflow** via CLI or API client. -2. **API validates** the request and creates a deployment in Prefect. -3. **Prefect schedules** the workflow and assigns it to a worker. -4. **Worker launches a container** to run the workflow. -5. **Results are stored** in Docker volumes and the database. -6. **Status updates** flow back through Prefect and the API to the user. +1. **User submits a workflow** via CLI or API client (with optional file upload). +2. **If file provided, API uploads** to MinIO and gets a `target_id`. +3. **API validates** the request and submits to Temporal. +4. **Temporal routes** the workflow to the appropriate vertical worker queue. +5. **Worker downloads target** from MinIO to local cache (if needed). +6. **Worker executes workflow** with pre-installed tools. +7. **Results are stored** in MinIO and metadata in PostgreSQL. +8. **Status updates** flow back through Temporal and the API to the user. ```mermaid sequenceDiagram participant User participant API - participant Prefect + participant MinIO + participant Temporal participant Worker - participant Container - participant Storage + participant Cache - User->>API: Submit workflow + User->>API: Submit workflow + file API->>API: Validate parameters - API->>Prefect: Create deployment - Prefect->>Worker: Schedule execution - Worker->>Container: Create and start - Container->>Container: Execute security tools - Container->>Storage: Store SARIF results - Worker->>Prefect: Update status - Prefect->>API: Workflow complete + API->>MinIO: Upload target file + MinIO-->>API: Return target_id + API->>Temporal: Submit workflow(target_id) + Temporal->>Worker: Route to vertical queue + Worker->>MinIO: Download target + MinIO-->>Worker: Stream file + Worker->>Cache: Store in local cache + Worker->>Worker: Execute security tools + Worker->>MinIO: Upload SARIF results + Worker->>Temporal: Update status + Temporal->>API: Workflow complete API->>User: Return results ``` @@ -149,25 +159,27 @@ sequenceDiagram ## How Do Services Communicate? -- **Internally:** FastAPI talks to Prefect via REST; Prefect coordinates with workers over HTTP; workers manage containers via the Docker Engine API. All core services use pooled connections to PostgreSQL. -- **Externally:** Users interact via CLI or API clients (HTTP REST). The MCP server can automate workflows via its own protocol. +- **Internally:** FastAPI talks to Temporal via gRPC; Temporal coordinates with workers over gRPC; workers access MinIO via S3 API. All core services use pooled connections to PostgreSQL. +- **Externally:** Users interact via CLI or API clients (HTTP REST). ## How Is Security Enforced? -- **Container Isolation:** Each workflow runs in its own Docker network, as a non-root user, with strict resource limits and only necessary volumes mounted. -- **Volume Security:** Source code is mounted read-only; results are written to dedicated, temporary volumes. -- **API Security:** All endpoints require API keys, validate inputs, enforce rate limits, and log requests for auditing. +- **Worker Isolation:** Each workflow runs in isolated vertical workers with pre-defined toolchains. +- **Storage Security:** Uploaded files stored in MinIO with lifecycle policies; read-only access by default. +- **API Security:** All endpoints validate inputs, enforce rate limits, and log requests for auditing. +- **No Host Access:** Workers access targets via MinIO, not host filesystem. ## How Does FuzzForge Scale? -- **Horizontally:** Add more Prefect workers to handle more workflows in parallel. Scale the database with read replicas and connection pooling. -- **Vertically:** Adjust CPU and memory limits for containers and services as needed. +- **Horizontally:** Add more vertical workers to handle more workflows in parallel. Scale specific worker types based on demand. +- **Vertically:** Adjust CPU and memory limits for workers and adjust concurrent activity limits. Example Docker Compose scaling: ```yaml services: - prefect-worker: + worker-rust: deploy: + replicas: 3 # Scale rust workers resources: limits: memory: 4G @@ -179,21 +191,22 @@ services: ## How Is It Deployed? -- **Development:** All services run via Docker Composeā€”backend, Prefect, workers, database, and registry. -- **Production:** Add load balancers, database clustering, and multiple worker instances for high availability. Health checks, metrics, and centralized logging support monitoring and troubleshooting. +- **Development:** All services run via Docker Composeā€”backend, Temporal, vertical workers, database, and MinIO. +- **Production:** Add load balancers, Temporal clustering, database replication, and multiple worker instances for high availability. Health checks, metrics, and centralized logging support monitoring and troubleshooting. ## How Is Configuration Managed? -- **Environment Variables:** Control core settings like database URLs, registry location, and Prefect API endpoints. -- **Service Discovery:** Docker Composeā€™s internal DNS lets services find each other by name, with consistent port mapping and health check endpoints. +- **Environment Variables:** Control core settings like database URLs, MinIO endpoints, and Temporal addresses. +- **Service Discovery:** Docker Compose's internal DNS lets services find each other by name, with consistent port mapping and health check endpoints. Example configuration: ```bash COMPOSE_PROJECT_NAME=fuzzforge DATABASE_URL=postgresql://postgres:postgres@postgres:5432/fuzzforge -PREFECT_API_URL=http://prefect-server:4200/api -DOCKER_REGISTRY=localhost:5001 -DOCKER_INSECURE_REGISTRY=true +TEMPORAL_ADDRESS=temporal:7233 +S3_ENDPOINT=http://minio:9000 +S3_ACCESS_KEY=fuzzforge +S3_SECRET_KEY=fuzzforge123 ``` ## How Are Failures Handled? @@ -203,9 +216,9 @@ DOCKER_INSECURE_REGISTRY=true ## Implementation Details -- **Tech Stack:** FastAPI (Python async), Prefect 3.x, Docker, Docker Compose, PostgreSQL (asyncpg), and Docker networking. -- **Performance:** Workflows start in 2ā€“5 seconds; results are retrieved quickly thanks to caching and database indexing. -- **Extensibility:** Add new workflows by deploying new Docker images; extend the API with new endpoints; configure storage backends as needed. +- **Tech Stack:** FastAPI (Python async), Temporal, MinIO, Docker, Docker Compose, PostgreSQL (asyncpg), and boto3 (S3 client). +- **Performance:** Workflows start immediately (workers are long-lived); results are retrieved quickly thanks to MinIO caching and database indexing. +- **Extensibility:** Add new workflows by mounting code; add new vertical workers with specialized toolchains; extend the API with new endpoints. --- diff --git a/docs/docs/concept/docker-containers.md b/docs/docs/concept/docker-containers.md index 3cff9d2..d010f44 100644 --- a/docs/docs/concept/docker-containers.md +++ b/docs/docs/concept/docker-containers.md @@ -22,58 +22,62 @@ FuzzForge relies on Docker containers for several key reasons: Every workflow in FuzzForge is executed inside a Docker container. Hereā€™s what that means in practice: -- **Workflow containers** are built from language-specific base images (like Python or Node.js), with security tools and workflow code pre-installed. -- **Infrastructure containers** (API server, Prefect, database) use official images and are configured for the platformā€™s needs. +- **Vertical worker containers** are built from language-specific base images with domain-specific security toolchains pre-installed (Android, Rust, Web, etc.). +- **Infrastructure containers** (API server, Temporal, MinIO, database) use official images and are configured for the platform's needs. -### Container Lifecycle: From Build to Cleanup +### Worker Lifecycle: From Build to Long-Running -The lifecycle of a workflow container looks like this: +The lifecycle of a vertical worker looks like this: -1. **Image Build:** A Docker image is built with all required tools and code. -2. **Image Push/Pull:** The image is pushed to (and later pulled from) a local or remote registry. -3. **Container Creation:** The container is created with the right volumes and environment. -4. **Execution:** The workflow runs inside the container. -5. **Result Storage:** Results are written to mounted volumes. -6. **Cleanup:** The container and any temporary data are removed. +1. **Image Build:** A Docker image is built with all required toolchains for the vertical. +2. **Worker Start:** The worker container starts as a long-lived process. +3. **Workflow Discovery:** Worker scans mounted `/app/toolbox` for workflows matching its vertical. +4. **Registration:** Workflows are registered with Temporal on the worker's task queue. +5. **Execution:** When a workflow is submitted, the worker downloads the target from MinIO and executes. +6. **Continuous Running:** Worker remains running, ready for the next workflow. ```mermaid graph TB - Build[Build Image] --> Push[Push to Registry] - Push --> Pull[Pull Image] - Pull --> Create[Create Container] - Create --> Mount[Mount Volumes] - Mount --> Start[Start Container] - Start --> Execute[Run Workflow] - Execute --> Results[Store Results] - Execute --> Stop[Stop Container] - Stop --> Cleanup[Cleanup Data] - Cleanup --> Remove[Remove Container] + Build[Build Worker Image] --> Start[Start Worker Container] + Start --> Mount[Mount Toolbox Volume] + Mount --> Discover[Discover Workflows] + Discover --> Register[Register with Temporal] + Register --> Ready[Worker Ready] + Ready --> Workflow[Workflow Submitted] + Workflow --> Download[Download Target from MinIO] + Download --> Execute[Execute Workflow] + Execute --> Upload[Upload Results to MinIO] + Upload --> Ready ``` --- -## Whatā€™s Inside a Workflow Container? +## What's Inside a Vertical Worker Container? -A typical workflow container is structured like this: +A typical vertical worker container is structured like this: -- **Base Image:** Usually a slim language image (e.g., `python:3.11-slim`). +- **Base Image:** Language-specific image (e.g., `python:3.11-slim`). - **System Dependencies:** Installed as needed (e.g., `git`, `curl`). -- **Security Tools:** Pre-installed (e.g., `semgrep`, `bandit`, `safety`). -- **Workflow Code:** Copied into the container. +- **Domain-Specific Toolchains:** Pre-installed (e.g., Rust: `AFL++`, `cargo-fuzz`; Android: `apktool`, `Frida`). +- **Temporal Python SDK:** For workflow execution. +- **Boto3:** For MinIO/S3 access. +- **Worker Script:** Discovers and registers workflows. - **Non-root User:** Created for execution. -- **Entrypoint:** Runs the workflow code. +- **Entrypoint:** Runs the worker discovery and registration loop. -Example Dockerfile snippet: +Example Dockerfile snippet for Rust worker: ```dockerfile FROM python:3.11-slim -RUN apt-get update && apt-get install -y git curl && rm -rf /var/lib/apt/lists/* -RUN pip install semgrep bandit safety -COPY ./toolbox /app/toolbox +RUN apt-get update && apt-get install -y git curl build-essential && rm -rf /var/lib/apt/lists/* +# Install AFL++, cargo, etc. +RUN pip install temporalio boto3 pydantic +COPY worker.py /app/ WORKDIR /app RUN useradd -m -u 1000 fuzzforge USER fuzzforge -CMD ["python", "-m", "toolbox.main"] +# Toolbox will be mounted as volume at /app/toolbox +CMD ["python", "worker.py"] ``` --- @@ -102,37 +106,42 @@ networks: ### Volume Types -- **Target Code Volume:** Mounts the code to be analyzed, read-only, into the container. -- **Result Volume:** Stores workflow results and artifacts, persists after container exit. -- **Temporary Volumes:** Used for scratch space, destroyed with the container. +- **Toolbox Volume:** Mounts the workflow code directory, read-only, for dynamic discovery. +- **Worker Cache:** Local cache for downloaded MinIO targets, with LRU eviction. +- **MinIO Data:** Persistent storage for uploaded targets and results (S3-compatible). Example volume mount: ```yaml volumes: - - "/host/path/to/code:/app/target:ro" - - "fuzzforge_prefect_storage:/app/prefect" + - "./toolbox:/app/toolbox:ro" # Workflow code + - "worker_cache:/cache" # Local cache + - "minio_data:/data" # MinIO storage ``` ### Volume Security -- **Read-only Mounts:** Prevent workflows from modifying source code. -- **Isolated Results:** Each workflow writes to its own result directory. -- **No Arbitrary Host Access:** Only explicitly mounted paths are accessible. +- **Read-only Toolbox:** Workflows cannot modify the mounted toolbox code. +- **Isolated Storage:** Each workflow's target is stored with a unique `target_id` in MinIO. +- **No Host Filesystem Access:** Workers access targets via MinIO, not host paths. +- **Automatic Cleanup:** MinIO lifecycle policies delete old targets after 7 days. --- -## How Are Images Built and Managed? +## How Are Worker Images Built and Managed? -- **Automated Builds:** Images are built and pushed to a local registry for development, or a secure registry for production. +- **Automated Builds:** Vertical worker images are built with specialized toolchains. - **Build Optimization:** Use layer caching, multi-stage builds, and minimal base images. -- **Versioning:** Use tags (`latest`, semantic versions, or SHA digests) to track images. +- **Versioning:** Use tags (`latest`, semantic versions) to track worker images. +- **Long-Lived:** Workers run continuously, not ephemeral per-workflow. -Example build and push: +Example build: ```bash -docker build -t localhost:5001/fuzzforge-static-analysis:latest . -docker push localhost:5001/fuzzforge-static-analysis:latest +cd workers/rust +docker build -t fuzzforge-worker-rust:latest . +# Or via docker-compose +docker-compose -f docker-compose.temporal.yaml build worker-rust ``` --- @@ -147,7 +156,7 @@ Example resource config: ```yaml services: - prefect-worker: + worker-rust: deploy: resources: limits: @@ -156,6 +165,8 @@ services: reservations: memory: 1G cpus: '0.5' + environment: + MAX_CONCURRENT_ACTIVITIES: 5 ``` --- @@ -172,7 +183,7 @@ Example security options: ```yaml services: - prefect-worker: + worker-rust: security_opt: - no-new-privileges:true cap_drop: @@ -188,8 +199,9 @@ services: ## How Is Performance Optimized? - **Image Layering:** Structure Dockerfiles for efficient caching. -- **Dependency Preinstallation:** Reduce startup time by pre-installing dependencies. -- **Warm Containers:** Optionally pre-create containers for faster workflow startup. +- **Pre-installed Toolchains:** All tools installed in worker image, zero setup time per workflow. +- **Long-Lived Workers:** Eliminate container startup overhead entirely. +- **Local Caching:** MinIO targets cached locally for repeated workflows. - **Horizontal Scaling:** Scale worker containers to handle more workflows in parallel. --- @@ -205,10 +217,10 @@ services: ## How Does This All Fit Into FuzzForge? -- **Prefect Workers:** Manage the full lifecycle of workflow containers. -- **API Integration:** Exposes container status, logs, and resource metrics. -- **Volume Management:** Ensures results and artifacts are collected and persisted. -- **Security and Resource Controls:** Enforced automatically for every workflow. +- **Temporal Workers:** Long-lived vertical workers execute workflows with pre-installed toolchains. +- **API Integration:** Exposes workflow status, logs, and resource metrics via Temporal. +- **MinIO Storage:** Ensures targets and results are stored, cached, and cleaned up automatically. +- **Security and Resource Controls:** Enforced automatically for every worker and workflow. --- diff --git a/docs/docs/concept/resource-management.md b/docs/docs/concept/resource-management.md new file mode 100644 index 0000000..ab56599 --- /dev/null +++ b/docs/docs/concept/resource-management.md @@ -0,0 +1,594 @@ +# Resource Management in FuzzForge + +FuzzForge uses a multi-layered approach to manage CPU, memory, and concurrency for workflow execution. This ensures stable operation, prevents resource exhaustion, and allows predictable performance. + +--- + +## Overview + +Resource limiting in FuzzForge operates at three levels: + +1. **Docker Container Limits** (Primary Enforcement) - Hard limits enforced by Docker +2. **Worker Concurrency Limits** - Controls parallel workflow execution +3. **Workflow Metadata** (Advisory) - Documents resource requirements + +--- + +## Worker Lifecycle Management (On-Demand Startup) + +**New in v0.7.0**: Workers now support on-demand startup/shutdown for optimal resource usage. + +### Architecture + +Workers are **pre-built** but **not auto-started**: + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ docker- ā”‚ Pre-built worker images +ā”‚ compose ā”‚ with profiles: ["workers", "ossfuzz"] +ā”‚ build ā”‚ restart: "no" +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā†“ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Workers ā”‚ Status: Exited (not running) +ā”‚ Pre-built ā”‚ RAM Usage: 0 MB +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā†“ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ ff workflow ā”‚ CLI detects required worker +ā”‚ run ā”‚ via /workflows/{name}/worker-info API +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā†“ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ docker ā”‚ docker start fuzzforge-worker-ossfuzz +ā”‚ start ā”‚ Wait for healthy status +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + ā†“ +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ Worker ā”‚ Status: Up +ā”‚ Running ā”‚ RAM Usage: ~1-2 GB +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ +``` + +### Resource Savings + +| State | Services Running | RAM Usage | +|-------|-----------------|-----------| +| **Idle** (no workflows) | Temporal, PostgreSQL, MinIO, Backend | ~1.2 GB | +| **Active** (1 workflow) | Core + 1 worker | ~3-5 GB | +| **Legacy** (all workers) | Core + all 5 workers | ~8 GB | + +**Savings: ~6-7GB RAM when idle** āœØ + +### Configuration + +Control via `.fuzzforge/config.yaml`: + +```yaml +workers: + auto_start_workers: true # Auto-start when needed + auto_stop_workers: false # Auto-stop after completion + worker_startup_timeout: 60 # Startup timeout (seconds) + docker_compose_file: null # Custom compose file path +``` + +Or via CLI flags: + +```bash +# Auto-start disabled +ff workflow run ossfuzz_campaign . --no-auto-start + +# Auto-stop enabled +ff workflow run ossfuzz_campaign . --wait --auto-stop +``` + +### Backend API + +New endpoint: `GET /workflows/{workflow_name}/worker-info` + +**Response**: +```json +{ + "workflow": "ossfuzz_campaign", + "vertical": "ossfuzz", + "worker_container": "fuzzforge-worker-ossfuzz", + "task_queue": "ossfuzz-queue", + "required": true +} +``` + +### SDK Integration + +```python +from fuzzforge_sdk import FuzzForgeClient + +client = FuzzForgeClient() +worker_info = client.get_workflow_worker_info("ossfuzz_campaign") +# Returns: {"vertical": "ossfuzz", "worker_container": "fuzzforge-worker-ossfuzz", ...} +``` + +### Manual Control + +```bash +# Start worker manually +docker start fuzzforge-worker-ossfuzz + +# Stop worker manually +docker stop fuzzforge-worker-ossfuzz + +# Check all worker statuses +docker ps -a --filter "name=fuzzforge-worker" +``` + +--- + +## Level 1: Docker Container Limits (Primary) + +Docker container limits are the **primary enforcement mechanism** for CPU and memory resources. These are configured in `docker-compose.temporal.yaml` and enforced by the Docker runtime. + +### Configuration + +```yaml +services: + worker-rust: + deploy: + resources: + limits: + cpus: '2.0' # Maximum 2 CPU cores + memory: 2G # Maximum 2GB RAM + reservations: + cpus: '0.5' # Minimum 0.5 CPU cores reserved + memory: 512M # Minimum 512MB RAM reserved +``` + +### How It Works + +- **CPU Limit**: Docker throttles CPU usage when the container exceeds the limit +- **Memory Limit**: Docker kills the container (OOM) if it exceeds the memory limit +- **Reservations**: Guarantees minimum resources are available to the worker + +### Example Configuration by Vertical + +Different verticals have different resource needs: + +**Rust Worker** (CPU-intensive fuzzing): +```yaml +worker-rust: + deploy: + resources: + limits: + cpus: '4.0' + memory: 4G +``` + +**Android Worker** (Memory-intensive emulation): +```yaml +worker-android: + deploy: + resources: + limits: + cpus: '2.0' + memory: 8G +``` + +**Web Worker** (Lightweight analysis): +```yaml +worker-web: + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G +``` + +### Monitoring Container Resources + +Check real-time resource usage: + +```bash +# Monitor all workers +docker stats + +# Monitor specific worker +docker stats fuzzforge-worker-rust + +# Output: +# CONTAINER CPU % MEM USAGE / LIMIT MEM % +# fuzzforge-worker-rust 85% 1.5GiB / 2GiB 75% +``` + +--- + +## Level 2: Worker Concurrency Limits + +The `MAX_CONCURRENT_ACTIVITIES` environment variable controls how many workflows can execute **simultaneously** on a single worker. + +### Configuration + +```yaml +services: + worker-rust: + environment: + MAX_CONCURRENT_ACTIVITIES: 5 + deploy: + resources: + limits: + memory: 2G +``` + +### How It Works + +- **Total Container Memory**: 2GB +- **Concurrent Workflows**: 5 +- **Memory per Workflow**: ~400MB (2GB Ć· 5) + +If a 6th workflow is submitted, it **waits in the Temporal queue** until one of the 5 running workflows completes. + +### Calculating Concurrency + +Use this formula to determine `MAX_CONCURRENT_ACTIVITIES`: + +``` +MAX_CONCURRENT_ACTIVITIES = Container Memory Limit / Estimated Workflow Memory +``` + +**Example:** +- Container limit: 4GB +- Workflow memory: ~800MB +- Concurrency: 4GB Ć· 800MB = **5 concurrent workflows** + +### Configuration Examples + +**High Concurrency (Lightweight Workflows)**: +```yaml +worker-web: + environment: + MAX_CONCURRENT_ACTIVITIES: 10 # Many small workflows + deploy: + resources: + limits: + memory: 2G # ~200MB per workflow +``` + +**Low Concurrency (Heavy Workflows)**: +```yaml +worker-rust: + environment: + MAX_CONCURRENT_ACTIVITIES: 2 # Few large workflows + deploy: + resources: + limits: + memory: 4G # ~2GB per workflow +``` + +### Monitoring Concurrency + +Check how many workflows are running: + +```bash +# View worker logs +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "Starting" + +# Check Temporal UI +# Open http://localhost:8233 +# Navigate to "Task Queues" ā†’ "rust" ā†’ See pending/running counts +``` + +--- + +## Level 3: Workflow Metadata (Advisory) + +Workflow metadata in `metadata.yaml` documents resource requirements, but these are **advisory only** (except for timeout). + +### Configuration + +```yaml +# backend/toolbox/workflows/security_assessment/metadata.yaml +requirements: + resources: + memory: "512Mi" # Estimated memory usage (advisory) + cpu: "500m" # Estimated CPU usage (advisory) + timeout: 1800 # Execution timeout in seconds (ENFORCED) +``` + +### What's Enforced vs Advisory + +| Field | Enforcement | Description | +|-------|-------------|-------------| +| `timeout` | āœ… **Enforced by Temporal** | Workflow killed if exceeds timeout | +| `memory` | āš ļø Advisory only | Documents expected memory usage | +| `cpu` | āš ļø Advisory only | Documents expected CPU usage | + +### Why Metadata Is Useful + +Even though `memory` and `cpu` are advisory, they're valuable for: + +1. **Capacity Planning**: Determine appropriate container limits +2. **Concurrency Tuning**: Calculate `MAX_CONCURRENT_ACTIVITIES` +3. **Documentation**: Communicate resource needs to users +4. **Scheduling Hints**: Future horizontal scaling logic + +### Timeout Enforcement + +The `timeout` field is **enforced by Temporal**: + +```python +# Temporal automatically cancels workflow after timeout +@workflow.defn +class SecurityAssessmentWorkflow: + @workflow.run + async def run(self, target_id: str): + # If this takes longer than metadata.timeout (1800s), + # Temporal will cancel the workflow + ... +``` + +**Check timeout in Temporal UI:** +1. Open http://localhost:8233 +2. Navigate to workflow execution +3. See "Timeout" in workflow details +4. If exceeded, status shows "TIMED_OUT" + +--- + +## Resource Management Best Practices + +### 1. Set Conservative Container Limits + +Start with lower limits and increase based on actual usage: + +```yaml +# Start conservative +worker-rust: + deploy: + resources: + limits: + cpus: '2.0' + memory: 2G + +# Monitor with: docker stats +# Increase if consistently hitting limits +``` + +### 2. Calculate Concurrency from Profiling + +Profile a single workflow first: + +```bash +# Run single workflow and monitor +docker stats fuzzforge-worker-rust + +# Note peak memory usage (e.g., 800MB) +# Calculate concurrency: 4GB Ć· 800MB = 5 +``` + +### 3. Set Realistic Timeouts + +Base timeouts on actual workflow duration: + +```yaml +# Static analysis: 5-10 minutes +timeout: 600 + +# Fuzzing: 1-24 hours +timeout: 86400 + +# Quick scans: 1-2 minutes +timeout: 120 +``` + +### 4. Monitor Resource Exhaustion + +Watch for these warning signs: + +```bash +# Check for OOM kills +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep -i "oom\|killed" + +# Check for CPU throttling +docker stats fuzzforge-worker-rust +# If CPU% consistently at limit ā†’ increase cpus + +# Check for memory pressure +docker stats fuzzforge-worker-rust +# If MEM% consistently >90% ā†’ increase memory +``` + +### 5. Use Vertical-Specific Configuration + +Different verticals have different needs: + +| Vertical | CPU Priority | Memory Priority | Typical Config | +|----------|--------------|-----------------|----------------| +| Rust Fuzzing | High | Medium | 4 CPUs, 4GB RAM | +| Android Analysis | Medium | High | 2 CPUs, 8GB RAM | +| Web Scanning | Low | Low | 1 CPU, 1GB RAM | +| Static Analysis | Medium | Medium | 2 CPUs, 2GB RAM | + +--- + +## Horizontal Scaling + +To handle more workflows, scale worker containers horizontally: + +```bash +# Scale rust worker to 3 instances +docker-compose -f docker-compose.temporal.yaml up -d --scale worker-rust=3 + +# Now you can run: +# - 3 workers Ɨ 5 concurrent activities = 15 workflows simultaneously +``` + +**How it works:** +- Temporal load balances across all workers on the same task queue +- Each worker has independent resource limits +- No shared state between workers + +--- + +## Troubleshooting Resource Issues + +### Issue: Workflows Stuck in "Running" State + +**Symptom:** Workflow shows RUNNING but makes no progress + +**Diagnosis:** +```bash +# Check worker is alive +docker-compose -f docker-compose.temporal.yaml ps worker-rust + +# Check worker resource usage +docker stats fuzzforge-worker-rust + +# Check for OOM kills +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep -i oom +``` + +**Solution:** +- Increase memory limit if worker was killed +- Reduce `MAX_CONCURRENT_ACTIVITIES` if overloaded +- Check worker logs for errors + +### Issue: "Too Many Pending Tasks" + +**Symptom:** Temporal shows many queued workflows + +**Diagnosis:** +```bash +# Check concurrent activities setting +docker exec fuzzforge-worker-rust env | grep MAX_CONCURRENT_ACTIVITIES + +# Check current workload +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "Starting" +``` + +**Solution:** +- Increase `MAX_CONCURRENT_ACTIVITIES` if resources allow +- Add more worker instances (horizontal scaling) +- Increase container resource limits + +### Issue: Workflow Timeout + +**Symptom:** Workflow shows "TIMED_OUT" in Temporal UI + +**Diagnosis:** +1. Check `metadata.yaml` timeout setting +2. Check Temporal UI for execution duration +3. Determine if timeout is appropriate + +**Solution:** +```yaml +# Increase timeout in metadata.yaml +requirements: + resources: + timeout: 3600 # Increased from 1800 +``` + +--- + +## Workspace Isolation and Cache Management + +FuzzForge uses workspace isolation to prevent concurrent workflows from interfering with each other. Each workflow run can have its own isolated workspace or share a common workspace based on the isolation mode. + +### Cache Directory Structure + +Workers cache downloaded targets locally to avoid repeated downloads: + +``` +/cache/ +ā”œā”€ā”€ {target_id_1}/ +ā”‚ ā”œā”€ā”€ {run_id_1}/ # Isolated mode +ā”‚ ā”‚ ā”œā”€ā”€ target # Downloaded tarball +ā”‚ ā”‚ ā””ā”€ā”€ workspace/ # Extracted files +ā”‚ ā”œā”€ā”€ {run_id_2}/ +ā”‚ ā”‚ ā”œā”€ā”€ target +ā”‚ ā”‚ ā””ā”€ā”€ workspace/ +ā”‚ ā””ā”€ā”€ workspace/ # Shared mode (no run_id) +ā”‚ ā””ā”€ā”€ ... +ā”œā”€ā”€ {target_id_2}/ +ā”‚ ā””ā”€ā”€ shared/ # Copy-on-write shared download +ā”‚ ā”œā”€ā”€ target +ā”‚ ā””ā”€ā”€ workspace/ +``` + +### Isolation Modes + +**Isolated Mode** (default for fuzzing): +- Each run gets `/cache/{target_id}/{run_id}/workspace/` +- Safe for concurrent execution +- Cleanup removes entire run directory + +**Shared Mode** (for read-only workflows): +- All runs share `/cache/{target_id}/workspace/` +- Efficient (downloads once) +- No cleanup (cache persists) + +**Copy-on-Write Mode**: +- Downloads to `/cache/{target_id}/shared/` +- Copies to `/cache/{target_id}/{run_id}/` per run +- Balances performance and isolation + +### Cache Limits + +Configure cache limits via environment variables: + +```yaml +worker-rust: + environment: + CACHE_DIR: /cache + CACHE_MAX_SIZE: 10GB # Maximum cache size before LRU eviction + CACHE_TTL: 7d # Time-to-live for cached files +``` + +### LRU Eviction + +When cache exceeds `CACHE_MAX_SIZE`, the least-recently-used files are automatically evicted: + +1. Worker tracks last access time for each cached target +2. When cache is full, oldest accessed files are removed first +3. Eviction runs periodically (every 30 minutes) + +### Monitoring Cache Usage + +Check cache size and cleanup logs: + +```bash +# Check cache size +docker exec fuzzforge-worker-rust du -sh /cache + +# Monitor cache evictions +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "Evicted from cache" + +# Check download vs cache hit rate +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep -E "Cache (HIT|MISS)" +``` + +See the [Workspace Isolation](/docs/concept/workspace-isolation) guide for complete details on isolation modes and when to use each. + +--- + +## Summary + +FuzzForge's resource management strategy: + +1. **Docker Container Limits**: Primary enforcement (CPU/memory hard limits) +2. **Concurrency Limits**: Controls parallel workflows per worker +3. **Workflow Metadata**: Advisory resource hints + enforced timeout +4. **Workspace Isolation**: Controls cache sharing and cleanup behavior + +**Key Takeaways:** +- Set conservative Docker limits and adjust based on monitoring +- Calculate `MAX_CONCURRENT_ACTIVITIES` from container memory Ć· workflow memory +- Use `docker stats` and Temporal UI to monitor resource usage +- Scale horizontally by adding more worker instances +- Set realistic timeouts based on actual workflow duration +- Choose appropriate isolation mode (isolated for fuzzing, shared for analysis) +- Monitor cache usage and adjust `CACHE_MAX_SIZE` as needed + +--- + +**Next Steps:** +- Review `docker-compose.temporal.yaml` resource configuration +- Profile your workflows to determine actual resource usage +- Adjust limits based on monitoring data +- Set up alerts for resource exhaustion diff --git a/docs/docs/concept/workflow.md b/docs/docs/concept/workflow.md index d46f9cc..854c31c 100644 --- a/docs/docs/concept/workflow.md +++ b/docs/docs/concept/workflow.md @@ -25,30 +25,31 @@ Hereā€™s how a workflow moves through the FuzzForge system: ```mermaid graph TB User[User/CLI/API] --> API[FuzzForge API] - API --> Prefect[Prefect Orchestrator] - Prefect --> Worker[Prefect Worker] - Worker --> Container[Docker Container] - Container --> Tools[Security Tools] + API --> MinIO[MinIO Storage] + API --> Temporal[Temporal Orchestrator] + Temporal --> Worker[Vertical Worker] + Worker --> MinIO + Worker --> Tools[Security Tools] Tools --> Results[SARIF Results] - Results --> Storage[Persistent Storage] + Results --> MinIO ``` **Key roles:** -- **User/CLI/API:** Submits and manages workflows. -- **FuzzForge API:** Validates, orchestrates, and tracks workflows. -- **Prefect Orchestrator:** Schedules and manages workflow execution. -- **Prefect Worker:** Runs the workflow in a Docker container. +- **User/CLI/API:** Submits workflows and uploads files. +- **FuzzForge API:** Validates, uploads targets, and tracks workflows. +- **Temporal Orchestrator:** Schedules and manages workflow execution. +- **Vertical Worker:** Long-lived worker with pre-installed security tools. +- **MinIO Storage:** Stores uploaded targets and results. - **Security Tools:** Perform the actual analysis. -- **Persistent Storage:** Stores results and artifacts. --- ## Workflow Lifecycle: From Idea to Results -1. **Design:** Choose tools, define integration logic, set up parameters, and build the Docker image. -2. **Deployment:** Build and push the image, register the workflow, and configure defaults. -3. **Execution:** User submits a workflow; parameters and target are validated; the workflow is scheduled and executed in a container; tools run as designed. -4. **Completion:** Results are collected, normalized, and stored; status is updated; temporary resources are cleaned up; results are made available via API/CLI. +1. **Design:** Choose tools, define integration logic, set up parameters, and specify the vertical worker. +2. **Deployment:** Create workflow code, add metadata with `vertical` field, mount as volume in worker. +3. **Execution:** User submits a workflow with file upload; file is stored in MinIO; workflow is routed to vertical worker; worker downloads target and executes; tools run as designed. +4. **Completion:** Results are collected, normalized, and stored in MinIO; status is updated; MinIO lifecycle policies clean up old files; results are made available via API/CLI. --- @@ -85,25 +86,25 @@ FuzzForge supports several workflow types, each optimized for a specific securit ## Data Flow and Storage -- **Input:** Target code and parameters are validated and mounted as read-only volumes. -- **Processing:** Tools are initialized and run (often in parallel); outputs are collected and normalized. -- **Output:** Results are stored in persistent volumes and indexed for fast retrieval; metadata is saved in the database; intermediate results may be cached for performance. +- **Input:** Target files uploaded via HTTP to MinIO; parameters validated and passed to Temporal. +- **Processing:** Worker downloads target from MinIO to local cache; tools are initialized and run (often in parallel); outputs are collected and normalized. +- **Output:** Results are stored in MinIO and indexed for fast retrieval; metadata is saved in PostgreSQL; targets cached locally for repeated workflows; lifecycle policies clean up after 7 days. --- ## Error Handling and Recovery -- **Tool-Level:** Timeouts, resource exhaustion, and crashes are handled gracefully; failed tools donā€™t stop the workflow. -- **Workflow-Level:** Container failures, volume issues, and network problems are detected and reported. -- **Recovery:** Automatic retries for transient errors; partial results are returned when possible; workflows degrade gracefully if some tools are unavailable. +- **Tool-Level:** Timeouts, resource exhaustion, and crashes are handled gracefully; failed tools don't stop the workflow. +- **Workflow-Level:** Worker failures, storage issues, and network problems are detected and reported by Temporal. +- **Recovery:** Automatic retries for transient errors via Temporal; partial results are returned when possible; workflows degrade gracefully if some tools are unavailable; MinIO ensures targets remain accessible. --- ## Performance and Optimization -- **Container Efficiency:** Docker images are layered and cached for fast startup; containers may be reused when safe. +- **Worker Efficiency:** Long-lived workers eliminate container startup overhead; pre-installed toolchains reduce setup time. - **Parallel Processing:** Independent tools run concurrently to maximize CPU usage and minimize wait times. -- **Caching:** Images, dependencies, and intermediate results are cached to avoid unnecessary recomputation. +- **Caching:** MinIO targets are cached locally; repeated workflows reuse cached targets; worker cache uses LRU eviction. --- diff --git a/docs/docs/concept/workspace-isolation.md b/docs/docs/concept/workspace-isolation.md new file mode 100644 index 0000000..9e1f797 --- /dev/null +++ b/docs/docs/concept/workspace-isolation.md @@ -0,0 +1,378 @@ +# Workspace Isolation + +FuzzForge's workspace isolation system ensures that concurrent workflow runs don't interfere with each other. This is critical for fuzzing and security analysis workloads where multiple workflows might process the same target simultaneously. + +--- + +## Why Workspace Isolation? + +### The Problem + +Without isolation, concurrent workflows accessing the same target would share the same cache directory: + +``` +/cache/{target_id}/workspace/ +``` + +This causes problems when: +- **Fuzzing workflows** modify corpus files and crash artifacts +- **Multiple runs** operate on the same target simultaneously +- **File conflicts** occur during read/write operations + +### The Solution + +FuzzForge implements configurable workspace isolation with three modes: + +1. **isolated** (default): Each run gets its own workspace +2. **shared**: All runs share the same workspace +3. **copy-on-write**: Download once, copy per run + +--- + +## Isolation Modes + +### Isolated Mode (Default) + +**Use for**: Fuzzing workflows, any workflow that modifies files + +**Cache path**: `/cache/{target_id}/{run_id}/workspace/` + +Each workflow run gets a completely isolated workspace directory. The target is downloaded to a run-specific path using the unique `run_id`. + +**Advantages:** +- āœ… Safe for concurrent execution +- āœ… No file conflicts +- āœ… Clean per-run state + +**Disadvantages:** +- āš ļø Downloads target for each run (higher bandwidth/storage) +- āš ļø No sharing of downloaded artifacts + +**Example workflows:** +- `atheris_fuzzing` - Modifies corpus, creates crash files +- `cargo_fuzzing` - Modifies corpus, generates artifacts + +**metadata.yaml:** +```yaml +name: atheris_fuzzing +workspace_isolation: "isolated" +``` + +**Cleanup behavior:** +Entire run directory `/cache/{target_id}/{run_id}/` is removed after workflow completes. + +--- + +### Shared Mode + +**Use for**: Read-only analysis workflows, security scanners + +**Cache path**: `/cache/{target_id}/workspace/` + +All workflow runs for the same target share a single workspace directory. The target is downloaded once and reused across runs. + +**Advantages:** +- āœ… Efficient (download once, use many times) +- āœ… Lower bandwidth and storage usage +- āœ… Faster startup (cache hit after first download) + +**Disadvantages:** +- āš ļø Not safe for workflows that modify files +- āš ļø Potential race conditions if workflows write + +**Example workflows:** +- `security_assessment` - Read-only file scanning and analysis +- `secret_detection` - Read-only secret scanning + +**metadata.yaml:** +```yaml +name: security_assessment +workspace_isolation: "shared" +``` + +**Cleanup behavior:** +No cleanup (workspace shared across runs). Cache persists until LRU eviction. + +--- + +### Copy-on-Write Mode + +**Use for**: Workflows that need isolation but benefit from shared initial download + +**Cache paths**: +- Shared download: `/cache/{target_id}/shared/target` +- Per-run copy: `/cache/{target_id}/{run_id}/workspace/` + +Target is downloaded once to a shared location, then copied for each run. + +**Advantages:** +- āœ… Download once (shared bandwidth) +- āœ… Isolated per-run workspace (safe for modifications) +- āœ… Balances performance and safety + +**Disadvantages:** +- āš ļø Copy overhead (disk I/O per run) +- āš ļø Higher storage usage than shared mode + +**metadata.yaml:** +```yaml +name: my_workflow +workspace_isolation: "copy-on-write" +``` + +**Cleanup behavior:** +Run-specific copies removed, shared download persists until LRU eviction. + +--- + +## How It Works + +### Activity Signature + +The `get_target` activity accepts isolation parameters: + +```python +from temporalio import workflow + +# In your workflow +target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "isolated"], # target_id, run_id, workspace_isolation + start_to_close_timeout=timedelta(minutes=5) +) +``` + +### Path Resolution + +Based on the isolation mode: + +```python +# Isolated mode +if workspace_isolation == "isolated": + cache_path = f"/cache/{target_id}/{run_id}/" + +# Shared mode +elif workspace_isolation == "shared": + cache_path = f"/cache/{target_id}/" + +# Copy-on-write mode +else: # copy-on-write + shared_path = f"/cache/{target_id}/shared/" + cache_path = f"/cache/{target_id}/{run_id}/" + # Download to shared_path, copy to cache_path +``` + +### Cleanup + +The `cleanup_cache` activity respects isolation mode: + +```python +await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "isolated"], # target_path, workspace_isolation + start_to_close_timeout=timedelta(minutes=1) +) +``` + +**Cleanup behavior by mode:** +- `isolated`: Removes `/cache/{target_id}/{run_id}/` entirely +- `shared`: Skips cleanup (shared across runs) +- `copy-on-write`: Removes run directory, keeps shared cache + +--- + +## Cache Management + +### Cache Directory Structure + +``` +/cache/ +ā”œā”€ā”€ {target_id_1}/ +ā”‚ ā”œā”€ā”€ {run_id_1}/ +ā”‚ ā”‚ ā”œā”€ā”€ target # Downloaded tarball +ā”‚ ā”‚ ā””ā”€ā”€ workspace/ # Extracted files +ā”‚ ā”œā”€ā”€ {run_id_2}/ +ā”‚ ā”‚ ā”œā”€ā”€ target +ā”‚ ā”‚ ā””ā”€ā”€ workspace/ +ā”‚ ā””ā”€ā”€ workspace/ # Shared mode (no run_id subdirectory) +ā”‚ ā””ā”€ā”€ ... +ā”œā”€ā”€ {target_id_2}/ +ā”‚ ā””ā”€ā”€ shared/ +ā”‚ ā”œā”€ā”€ target # Copy-on-write shared download +ā”‚ ā””ā”€ā”€ workspace/ +``` + +### LRU Eviction + +When cache exceeds the configured limit (default: 10GB), least-recently-used files are evicted automatically. + +**Configuration:** +```yaml +# In worker environment +CACHE_DIR: /cache +CACHE_MAX_SIZE: 10GB +CACHE_TTL: 7d +``` + +**Eviction policy:** +- Tracks last access time for each cached target +- When cache is full, removes oldest accessed files first +- Cleanup runs periodically (every 30 minutes) + +--- + +## Choosing the Right Mode + +### Decision Matrix + +| Workflow Type | Modifies Files? | Concurrent Runs? | Recommended Mode | +|---------------|----------------|------------------|------------------| +| Fuzzing (AFL, libFuzzer, Atheris) | āœ… Yes | āœ… Yes | **isolated** | +| Static Analysis | āŒ No | āœ… Yes | **shared** | +| Secret Scanning | āŒ No | āœ… Yes | **shared** | +| File Modification | āœ… Yes | āŒ No | **isolated** | +| Large Downloads | āŒ No | āœ… Yes | **copy-on-write** | + +### Guidelines + +**Use `isolated` when:** +- Workflow modifies files (corpus, crashes, logs) +- Fuzzing or dynamic analysis +- Concurrent runs must not interfere + +**Use `shared` when:** +- Workflow only reads files +- Static analysis or scanning +- Want to minimize bandwidth/storage + +**Use `copy-on-write` when:** +- Workflow modifies files but target is large (>100MB) +- Want isolation but minimize download overhead +- Balance between shared and isolated + +--- + +## Configuration + +### In Workflow Metadata + +Document the isolation mode in `metadata.yaml`: + +```yaml +name: atheris_fuzzing +version: "1.0.0" +vertical: python + +# Workspace isolation mode +# - "isolated" (default): Each run gets own workspace +# - "shared": All runs share workspace (read-only workflows) +# - "copy-on-write": Download once, copy per run +workspace_isolation: "isolated" +``` + +### In Workflow Code + +Pass isolation mode to storage activities: + +```python +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self, target_id: str) -> Dict[str, Any]: + # Get run ID for isolation + run_id = workflow.info().run_id + + # Download target with isolation + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "isolated"], + start_to_close_timeout=timedelta(minutes=5) + ) + + # ... workflow logic ... + + # Cleanup with same isolation mode + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "isolated"], + start_to_close_timeout=timedelta(minutes=1) + ) +``` + +--- + +## Troubleshooting + +### Issue: Workflows interfere with each other + +**Symptom:** Fuzzing crashes from one run appear in another + +**Diagnosis:** +```bash +# Check workspace paths in logs +docker logs fuzzforge-worker-python | grep "User code downloaded" + +# Should see run-specific paths: +# āœ… /cache/abc-123/run-xyz-456/workspace (isolated) +# āŒ /cache/abc-123/workspace (shared - problem for fuzzing) +``` + +**Solution:** Change `workspace_isolation` to `"isolated"` in metadata.yaml + +### Issue: High bandwidth usage + +**Symptom:** Target downloaded repeatedly for same target_id + +**Diagnosis:** +```bash +# Check MinIO downloads in logs +docker logs fuzzforge-worker-python | grep "downloading from MinIO" + +# If many downloads for same target_id with shared workflow: +# Problem is using "isolated" mode for read-only workflow +``` + +**Solution:** Change to `"shared"` mode for read-only workflows + +### Issue: Cache fills up quickly + +**Symptom:** Disk space consumed by /cache directory + +**Diagnosis:** +```bash +# Check cache size +docker exec fuzzforge-worker-python du -sh /cache + +# Check LRU settings +docker exec fuzzforge-worker-python env | grep CACHE +``` + +**Solution:** +- Increase `CACHE_MAX_SIZE` environment variable +- Use `shared` mode for read-only workflows +- Decrease `CACHE_TTL` for faster eviction + +--- + +## Summary + +FuzzForge's workspace isolation system provides: + +1. **Safe concurrent execution** for fuzzing and analysis workflows +2. **Three isolation modes** to balance safety vs efficiency +3. **Automatic cache management** with LRU eviction +4. **Per-workflow configuration** via metadata.yaml + +**Key Takeaways:** +- Use `isolated` (default) for workflows that modify files +- Use `shared` for read-only analysis workflows +- Use `copy-on-write` to balance isolation and bandwidth +- Configure via `workspace_isolation` field in metadata.yaml +- Workers automatically handle download, extraction, and cleanup + +--- + +**Next Steps:** +- Review your workflows and set appropriate isolation modes +- Monitor cache usage with `docker exec fuzzforge-worker-python du -sh /cache` +- Adjust `CACHE_MAX_SIZE` if needed for your workload diff --git a/docs/docs/how-to/cicd-integration.md b/docs/docs/how-to/cicd-integration.md new file mode 100644 index 0000000..377cad2 --- /dev/null +++ b/docs/docs/how-to/cicd-integration.md @@ -0,0 +1,550 @@ +# CI/CD Integration Guide + +This guide shows you how to integrate FuzzForge into your CI/CD pipeline for automated security testing on every commit, pull request, or scheduled run. + +--- + +## Overview + +FuzzForge can run entirely inside CI containers (GitHub Actions, GitLab CI, etc.) with no external infrastructure required. The complete FuzzForge stackā€”Temporal, PostgreSQL, MinIO, Backend, and workersā€”starts automatically when needed and cleans up after execution. + +### Key Benefits + +āœ… **Zero Infrastructure**: No servers to maintain +āœ… **Ephemeral**: Fresh environment per run +āœ… **Resource Efficient**: On-demand workers (v0.7.0) save ~6-7GB RAM +āœ… **Fast Feedback**: Fail builds on critical/high findings +āœ… **Standards Compliant**: SARIF export for GitHub Security / GitLab SAST + +--- + +## Prerequisites + +### Required +- **CI Runner**: Ubuntu with Docker support +- **RAM**: At least 4GB available (7GB on GitHub Actions) +- **Startup Time**: ~60-90 seconds + +### Optional +- **jq**: For merging Docker daemon config (auto-installed in examples) +- **Python 3.11+**: For FuzzForge CLI + +--- + +## Quick Start + +### 1. Add Startup Scripts + +FuzzForge provides helper scripts to configure Docker and start services: + +```bash +# Start FuzzForge (configure Docker, start services, wait for health) +bash scripts/ci-start.sh + +# Stop and cleanup after execution +bash scripts/ci-stop.sh +``` + +### 2. Install CLI + +```bash +pip install ./cli +``` + +### 3. Initialize Project + +```bash +ff init --api-url http://localhost:8000 --name "CI Security Scan" +``` + +### 4. Run Workflow + +```bash +# Run and fail on error findings +ff workflow run security_assessment . \ + --wait \ + --fail-on error \ + --export-sarif results.sarif +``` + +--- + +## Deployment Models + +FuzzForge supports two CI/CD deployment models: + +### Option A: Ephemeral (Recommended) + +**Everything runs inside the CI container for each job.** + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ GitHub Actions Runner ā”‚ +ā”‚ ā”‚ +ā”‚ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”‚ +ā”‚ ā”‚ FuzzForge Stack ā”‚ ā”‚ +ā”‚ ā”‚ ā€¢ Temporal ā”‚ ā”‚ +ā”‚ ā”‚ ā€¢ PostgreSQL ā”‚ ā”‚ +ā”‚ ā”‚ ā€¢ MinIO ā”‚ ā”‚ +ā”‚ ā”‚ ā€¢ Backend ā”‚ ā”‚ +ā”‚ ā”‚ ā€¢ Workers (on-demand) ā”‚ ā”‚ +ā”‚ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā”‚ +ā”‚ ā”‚ +ā”‚ ff workflow run ... ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ +``` + +**Pros:** +- No infrastructure to maintain +- Complete isolation per run +- Works on GitHub/GitLab free tier + +**Cons:** +- 60-90s startup time per run +- Limited to runner resources + +**Best For:** Open source projects, infrequent scans, PR checks + +### Option B: Persistent Backend + +**Backend runs on a separate server, CLI connects remotely.** + +``` +ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” +ā”‚ CI Runner ā”‚ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā–¶ā”‚ FuzzForge Server ā”‚ +ā”‚ (ff CLI) ā”‚ HTTPS ā”‚ (self-hosted) ā”‚ +ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ +``` + +**Pros:** +- No startup time +- More resources +- Faster execution + +**Cons:** +- Requires infrastructure +- Needs API tokens + +**Best For:** Large teams, frequent scans, long fuzzing campaigns + +--- + +## GitHub Actions Integration + +### Complete Example + +See `.github/workflows/examples/security-scan.yml` for a full working example. + +**Basic workflow:** + +```yaml +name: Security Scan + +on: [pull_request, push] + +jobs: + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Start FuzzForge + run: bash scripts/ci-start.sh + + - name: Install CLI + run: pip install ./cli + + - name: Security Scan + run: | + ff init --api-url http://localhost:8000 + ff workflow run security_assessment . \ + --wait \ + --fail-on error \ + --export-sarif results.sarif + + - name: Upload SARIF + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif + + - name: Cleanup + if: always() + run: bash scripts/ci-stop.sh +``` + +### GitHub Security Tab Integration + +Upload SARIF results to see findings directly in GitHub: + +```yaml +- name: Upload SARIF to GitHub Security + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif +``` + +Findings appear in: +- **Security** tab ā†’ **Code scanning alerts** +- Pull request annotations +- Commit status checks + +--- + +## GitLab CI Integration + +### Complete Example + +See `.gitlab-ci.example.yml` for a full working example. + +**Basic pipeline:** + +```yaml +stages: + - security + +variables: + FUZZFORGE_API_URL: "http://localhost:8000" + +security:scan: + image: docker:24 + services: + - docker:24-dind + before_script: + - apk add bash python3 py3-pip + - bash scripts/ci-start.sh + - pip3 install ./cli --break-system-packages + - ff init --api-url $FUZZFORGE_API_URL + script: + - ff workflow run security_assessment . --wait --fail-on error --export-sarif results.sarif + artifacts: + reports: + sast: results.sarif + after_script: + - bash scripts/ci-stop.sh +``` + +### GitLab SAST Dashboard Integration + +The `reports: sast:` section automatically integrates with GitLab's Security Dashboard. + +--- + +## CLI Flags for CI/CD + +### `--fail-on` + +Fail the build if findings match specified SARIF severity levels. + +**Syntax:** +```bash +--fail-on error,warning,note,info,all,none +``` + +**SARIF Levels:** +- `error` - Critical security issues (fail build) +- `warning` - Potential security issues (may fail build) +- `note` - Informational findings (typically don't fail) +- `info` - Additional context (rarely blocks) +- `all` - Any finding (strictest) +- `none` - Never fail (report only) + +**Examples:** +```bash +# Fail on errors only (recommended for CI) +--fail-on error + +# Fail on errors or warnings +--fail-on error,warning + +# Fail on any finding (strictest) +--fail-on all + +# Never fail, just report (useful for monitoring) +--fail-on none +``` + +**Common Patterns:** +- **PR checks**: `--fail-on error` (block critical issues) +- **Release gates**: `--fail-on error,warning` (stricter) +- **Nightly scans**: `--fail-on none` (monitoring only) +- **Security audit**: `--fail-on all` (maximum strictness) + +**Exit Codes:** +- `0` - No blocking findings +- `1` - Found blocking findings or error + +### `--export-sarif` + +Export SARIF results to a file after workflow completion. + +**Syntax:** +```bash +--export-sarif +``` + +**Example:** +```bash +ff workflow run security_assessment . \ + --wait \ + --export-sarif results.sarif +``` + +### `--wait` + +Wait for workflow execution to complete (required for CI/CD). + +**Example:** +```bash +ff workflow run security_assessment . --wait +``` + +Without `--wait`, the command returns immediately and the workflow runs in the background. + +--- + +## Common Workflows + +### PR Security Gate + +Block PRs with critical/high findings: + +```yaml +on: pull_request + +jobs: + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash scripts/ci-start.sh + - run: pip install ./cli + - run: | + ff init --api-url http://localhost:8000 + ff workflow run security_assessment . --wait --fail-on error + - if: always() + run: bash scripts/ci-stop.sh +``` + +### Secret Detection (Zero Tolerance) + +Fail on ANY exposed secrets: + +```bash +ff workflow run secret_detection . --wait --fail-on all +``` + +### Nightly Fuzzing (Report Only) + +Run long fuzzing campaigns without failing the build: + +```yaml +on: + schedule: + - cron: '0 2 * * *' # 2 AM daily + +jobs: + fuzzing: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - run: bash scripts/ci-start.sh + - run: pip install ./cli + - run: | + ff init --api-url http://localhost:8000 + ff workflow run atheris_fuzzing . \ + max_iterations=100000000 \ + timeout_seconds=7200 \ + --wait \ + --export-sarif fuzzing-results.sarif + continue-on-error: true + - if: always() + run: bash scripts/ci-stop.sh +``` + +### Release Gate + +Block releases with ANY security findings: + +```yaml +on: + push: + tags: + - 'v*' + +jobs: + release-security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash scripts/ci-start.sh + - run: pip install ./cli + - run: | + ff init --api-url http://localhost:8000 + ff workflow run security_assessment . --wait --fail-on all +``` + +--- + +## Performance Optimization + +### Startup Time + +**Current:** ~60-90 seconds +**Breakdown:** +- Docker daemon restart: 10-15s +- docker-compose up: 30-40s +- Health check wait: 20-30s + +**Tips to reduce:** +1. Use `docker-compose.ci.yml` (optional, see below) +2. Cache Docker layers (GitHub Actions) +3. Use self-hosted runners (persistent Docker) + +### Optional: CI-Optimized Compose File + +Create `docker-compose.ci.yml`: + +```yaml +version: '3.8' + +services: + postgresql: + # Use in-memory storage (faster, ephemeral) + tmpfs: + - /var/lib/postgresql/data + command: postgres -c fsync=off -c full_page_writes=off + + minio: + # Use in-memory storage + tmpfs: + - /data + + temporal: + healthcheck: + # More frequent health checks + interval: 5s + retries: 10 +``` + +**Usage:** +```bash +docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d +``` + +--- + +## Troubleshooting + +### "Permission denied" connecting to Docker socket + +**Solution:** Add user to docker group or use `sudo`. + +```bash +# GitHub Actions (already has permissions) +# GitLab CI: use docker:dind service +``` + +### "Connection refused to localhost:8000" + +**Problem:** Services not healthy yet. + +**Solution:** Increase health check timeout in `ci-start.sh`: + +```bash +timeout 180 bash -c 'until curl -sf http://localhost:8000/health; do sleep 3; done' +``` + +### "Out of disk space" + +**Problem:** Docker volumes filling up. + +**Solution:** Cleanup in `after_script`: + +```bash +after_script: + - bash scripts/ci-stop.sh + - docker system prune -af --volumes +``` + +### Worker not starting + +**Problem:** Worker container exists but not running. + +**Solution:** Workers are pre-built but start on-demand (v0.7.0). If a workflow fails immediately, check: + +```bash +docker logs fuzzforge-worker- +``` + +--- + +## Best Practices + +1. **Always use `--wait`** in CI/CD pipelines +2. **Set appropriate `--fail-on` levels** for your use case: + - PR checks: `error` (block critical issues) + - Release gates: `error,warning` (stricter) + - Nightly scans: Don't use (report only) +3. **Export SARIF** to integrate with security dashboards +4. **Set timeouts** on CI jobs to prevent hanging +5. **Use artifacts** to preserve findings for review +6. **Cleanup always** with `if: always()` or `after_script` + +--- + +## Advanced: Persistent Backend Setup + +For high-frequency usage, deploy FuzzForge on a dedicated server: + +### 1. Deploy FuzzForge Server + +```bash +# On your CI server +git clone https://github.com/FuzzingLabs/fuzzforge_ai.git +cd fuzzforge_ai +docker-compose up -d +``` + +### 2. Generate API Token (Future Feature) + +```bash +# This will be available in a future release +docker exec fuzzforge-backend python -c " +from src.auth import generate_token +print(generate_token(name='github-actions')) +" +``` + +### 3. Configure CI to Use Remote Backend + +```yaml +env: + FUZZFORGE_API_URL: https://fuzzforge.company.com + FUZZFORGE_API_TOKEN: ${{ secrets.FUZZFORGE_TOKEN }} + +steps: + - run: pip install fuzzforge-cli + - run: ff workflow run security_assessment . --wait --fail-on error +``` + +**Note:** Authentication is not yet implemented (v0.7.0). Use network isolation or VPN for now. + +--- + +## Examples + +- **GitHub Actions**: `.github/workflows/examples/security-scan.yml` +- **GitLab CI**: `.gitlab-ci.example.yml` +- **Startup Script**: `scripts/ci-start.sh` +- **Cleanup Script**: `scripts/ci-stop.sh` + +--- + +## Support + +- **Documentation**: [https://docs.fuzzforge.io](https://docs.fuzzforge.io) +- **Issues**: [GitHub Issues](https://github.com/FuzzingLabs/fuzzforge_ai/issues) +- **Discussions**: [GitHub Discussions](https://github.com/FuzzingLabs/fuzzforge_ai/discussions) diff --git a/docs/docs/how-to/create-workflow.md b/docs/docs/how-to/create-workflow.md index cd32534..3a5c814 100644 --- a/docs/docs/how-to/create-workflow.md +++ b/docs/docs/how-to/create-workflow.md @@ -9,18 +9,18 @@ This guide will walk you through the process of creating a custom security analy Before you start, make sure you have: - A working FuzzForge development environment (see [Contributing](/reference/contributing.md)) -- Familiarity with Python (async/await), Docker, and Prefect 3 +- Familiarity with Python (async/await), Docker, and Temporal - At least one custom or built-in module to use in your workflow --- ## Step 1: Understand Workflow Architecture -A FuzzForge workflow is a Prefect 3 flow that: +A FuzzForge workflow is a Temporal workflow that: -- Runs in an isolated Docker container +- Runs inside a long-lived vertical worker container (pre-built with toolchains) - Orchestrates one or more analysis modules (scanner, analyzer, reporter, etc.) -- Handles secure volume mounting for code and results +- Downloads targets from MinIO (S3-compatible storage) automatically - Produces standardized SARIF output - Supports configurable parameters and resource limits @@ -28,9 +28,9 @@ A FuzzForge workflow is a Prefect 3 flow that: ``` backend/toolbox/workflows/{workflow_name}/ -ā”œā”€ā”€ workflow.py # Main workflow definition (Prefect flow) -ā”œā”€ā”€ Dockerfile # Container image definition -ā”œā”€ā”€ metadata.yaml # Workflow metadata and configuration +ā”œā”€ā”€ workflow.py # Main workflow definition (Temporal workflow) +ā”œā”€ā”€ activities.py # Workflow activities (optional) +ā”œā”€ā”€ metadata.yaml # Workflow metadata and configuration (must include vertical field) ā””ā”€ā”€ requirements.txt # Additional Python dependencies (optional) ``` @@ -48,6 +48,7 @@ version: "1.0.0" description: "Analyzes project dependencies for security vulnerabilities" author: "FuzzingLabs Security Team" category: "comprehensive" +vertical: "web" # REQUIRED: Which vertical worker to use (rust, android, web, etc.) tags: - "dependency-scanning" - "vulnerability-analysis" @@ -63,10 +64,6 @@ requirements: parameters: type: object properties: - target_path: - type: string - default: "/workspace" - description: "Path to analyze" scan_dev_dependencies: type: boolean description: "Include development dependencies" @@ -85,36 +82,63 @@ output_schema: description: "Scan execution summary" ``` +**Important:** The `vertical` field determines which worker runs your workflow. Ensure the worker has the required tools installed. + +### Workspace Isolation + +Add the `workspace_isolation` field to control how workflow runs share or isolate workspaces: + +```yaml +# Workspace isolation mode (system-level configuration) +# - "isolated" (default): Each workflow run gets its own isolated workspace +# - "shared": All runs share the same workspace (for read-only workflows) +# - "copy-on-write": Download once, copy for each run +workspace_isolation: "isolated" +``` + +**Choosing the right mode:** + +- **`isolated`** (default) - For fuzzing workflows that modify files (corpus, crashes) + - Example: `atheris_fuzzing`, `cargo_fuzzing` + - Safe for concurrent execution + +- **`shared`** - For read-only analysis workflows + - Example: `security_assessment`, `secret_detection` + - Efficient (downloads once, reuses cache) + +- **`copy-on-write`** - For large targets that need isolation + - Downloads once, copies per run + - Balances performance and isolation + +See the [Workspace Isolation](/docs/concept/workspace-isolation) guide for details. + --- ## Step 3: Add Live Statistics to Your Workflow šŸš¦ -Want real-time progress and stats for your workflow? FuzzForge supports live statistics reporting using Prefect and structured logging. This lets users (and the platform) monitor workflow progress, see live updates, and stream stats via API or WebSocket. +Want real-time progress and stats for your workflow? FuzzForge supports live statistics reporting using Temporal workflow logging. This lets users (and the platform) monitor workflow progress, see live updates, and stream stats via API or WebSocket. ### 1. Import Required Dependencies ```python -from prefect import task, get_run_context +from temporalio import workflow, activity import logging logger = logging.getLogger(__name__) ``` -### 2. Create a Statistics Callback Function +### 2. Create a Statistics Callback in Activity -Add a callback that logs structured stats updates: +Add a callback that logs structured stats updates in your activity: ```python -@task(name="my_workflow_task") -async def my_workflow_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: - # Get run context for statistics reporting - try: - context = get_run_context() - run_id = str(context.flow_run.id) - logger.info(f"Running task for flow run: {run_id}") - except Exception: - run_id = None - logger.warning("Could not get run context for statistics") +@activity.defn +async def my_workflow_activity(target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: + # Get activity info for run tracking + info = activity.info() + run_id = info.workflow_id + + logger.info(f"Running activity for workflow: {run_id}") # Define callback function for live statistics async def stats_callback(stats_data: Dict[str, Any]): @@ -124,7 +148,7 @@ async def my_workflow_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, logger.info("LIVE_STATS", extra={ "stats_type": "live_stats", # Type of statistics "workflow_type": "my_workflow", # Your workflow name - "run_id": stats_data.get("run_id"), + "run_id": run_id, # Add your custom statistics fields here: "progress": stats_data.get("progress", 0), @@ -138,7 +162,7 @@ async def my_workflow_task(workspace: Path, config: Dict[str, Any]) -> Dict[str, # Pass callback to your module/processor processor = MyWorkflowModule() - result = await processor.execute(config, workspace, stats_callback=stats_callback) + result = await processor.execute(config, target_path, stats_callback=stats_callback) return result.dict() ``` @@ -224,15 +248,16 @@ Live statistics automatically appear in: #### Example: Adding Stats to a Security Scanner ```python -async def security_scan_task(workspace: Path, config: Dict[str, Any]): - context = get_run_context() - run_id = str(context.flow_run.id) +@activity.defn +async def security_scan_activity(target_path: str, config: Dict[str, Any]): + info = activity.info() + run_id = info.workflow_id async def stats_callback(stats_data): logger.info("LIVE_STATS", extra={ "stats_type": "scan_progress", "workflow_type": "security_scan", - "run_id": stats_data.get("run_id"), + "run_id": run_id, "files_scanned": stats_data.get("files_scanned", 0), "vulnerabilities_found": stats_data.get("vulnerabilities_found", 0), "scan_percentage": stats_data.get("scan_percentage", 0.0), @@ -241,7 +266,7 @@ async def security_scan_task(workspace: Path, config: Dict[str, Any]): }) scanner = SecurityScannerModule() - return await scanner.execute(config, workspace, stats_callback=stats_callback) + return await scanner.execute(config, target_path, stats_callback=stats_callback) ``` With these steps, your workflow will provide rich, real-time feedback to users and the FuzzForge platformā€”making automation more transparent and interactive! @@ -250,95 +275,182 @@ With these steps, your workflow will provide rich, real-time feedback to users a ## Step 4: Implement the Workflow Logic -Create a `workflow.py` file. This is where you define your Prefect flow and tasks. +Create a `workflow.py` file. This is where you define your Temporal workflow and activities. Example (simplified): ```python from pathlib import Path from typing import Dict, Any -from prefect import flow, task +from temporalio import workflow, activity +from datetime import timedelta from src.toolbox.modules.dependency_scanner import DependencyScanner from src.toolbox.modules.vulnerability_analyzer import VulnerabilityAnalyzer from src.toolbox.modules.reporter import SARIFReporter -@task -async def scan_dependencies(workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: +@activity.defn +async def scan_dependencies(target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: scanner = DependencyScanner() - return (await scanner.execute(config, workspace)).dict() + return (await scanner.execute(config, target_path)).dict() -@task -async def analyze_vulnerabilities(dependencies: Dict[str, Any], workspace: Path, config: Dict[str, Any]) -> Dict[str, Any]: +@activity.defn +async def analyze_vulnerabilities(dependencies: Dict[str, Any], target_path: str, config: Dict[str, Any]) -> Dict[str, Any]: analyzer = VulnerabilityAnalyzer() analyzer_config = {**config, 'dependencies': dependencies.get('findings', [])} - return (await analyzer.execute(analyzer_config, workspace)).dict() + return (await analyzer.execute(analyzer_config, target_path)).dict() -@task -async def generate_report(dep_results: Dict[str, Any], vuln_results: Dict[str, Any], config: Dict[str, Any], workspace: Path) -> Dict[str, Any]: +@activity.defn +async def generate_report(dep_results: Dict[str, Any], vuln_results: Dict[str, Any], config: Dict[str, Any]) -> Dict[str, Any]: reporter = SARIFReporter() all_findings = dep_results.get("findings", []) + vuln_results.get("findings", []) reporter_config = {**config, "findings": all_findings} - return (await reporter.execute(reporter_config, workspace)).dict().get("sarif", {}) + return (await reporter.execute(reporter_config, None)).dict().get("sarif", {}) -@flow(name="dependency_analysis") -async def main_flow( - target_path: str = "/workspace", - scan_dev_dependencies: bool = True, - vulnerability_threshold: str = "medium" -) -> Dict[str, Any]: - workspace = Path(target_path) - scanner_config = {"scan_dev_dependencies": scan_dev_dependencies} - analyzer_config = {"vulnerability_threshold": vulnerability_threshold} - reporter_config = {} +@workflow.defn +class DependencyAnalysisWorkflow: + @workflow.run + async def run( + self, + target_id: str, # Target file ID from MinIO (downloaded by worker automatically) + scan_dev_dependencies: bool = True, + vulnerability_threshold: str = "medium" + ) -> Dict[str, Any]: + workflow.logger.info(f"Starting dependency analysis for target: {target_id}") - dep_results = await scan_dependencies(workspace, scanner_config) - vuln_results = await analyze_vulnerabilities(dep_results, workspace, analyzer_config) - sarif_report = await generate_report(dep_results, vuln_results, reporter_config, workspace) - return sarif_report + # Get run ID for workspace isolation + run_id = workflow.info().run_id + + # Worker downloads target from MinIO with isolation + target_path = await workflow.execute_activity( + "get_target", + args=[target_id, run_id, "shared"], # target_id, run_id, workspace_isolation + start_to_close_timeout=timedelta(minutes=5) + ) + + scanner_config = {"scan_dev_dependencies": scan_dev_dependencies} + analyzer_config = {"vulnerability_threshold": vulnerability_threshold} + + # Execute activities with retries and timeouts + dep_results = await workflow.execute_activity( + scan_dependencies, + args=[target_path, scanner_config], + start_to_close_timeout=timedelta(minutes=10), + retry_policy=workflow.RetryPolicy(maximum_attempts=3) + ) + + vuln_results = await workflow.execute_activity( + analyze_vulnerabilities, + args=[dep_results, target_path, analyzer_config], + start_to_close_timeout=timedelta(minutes=10), + retry_policy=workflow.RetryPolicy(maximum_attempts=3) + ) + + sarif_report = await workflow.execute_activity( + generate_report, + args=[dep_results, vuln_results, {}], + start_to_close_timeout=timedelta(minutes=5), + retry_policy=workflow.RetryPolicy(maximum_attempts=3) + ) + + # Cleanup cache (respects isolation mode) + await workflow.execute_activity( + "cleanup_cache", + args=[target_path, "shared"], # target_path, workspace_isolation + start_to_close_timeout=timedelta(minutes=1) + ) + + workflow.logger.info("Dependency analysis completed") + return sarif_report ``` +**Key Temporal Workflow Concepts:** +- Use `@workflow.defn` class decorator to define workflows +- Use `@activity.defn` decorator for activity functions +- Call `get_target` activity to download targets from MinIO with workspace isolation +- Use `workflow.execute_activity()` with explicit timeouts and retry policies +- Use `workflow.logger` for logging (appears in Temporal UI and backend logs) +- Call `cleanup_cache` activity at end to clean up workspace + --- -## Step 5: Create the Dockerfile +## Step 5: No Dockerfile Needed! šŸŽ‰ -Your workflow runs in a container. Create a `Dockerfile`: +**Good news:** You don't need to create a Dockerfile for your workflow. Workflows run inside pre-built **vertical worker containers** that already have toolchains installed. -```dockerfile -FROM python:3.11-slim -WORKDIR /app -RUN apt-get update && apt-get install -y git curl && rm -rf /var/lib/apt/lists/* -COPY ../../../pyproject.toml ./ -COPY ../../../uv.lock ./ -RUN pip install uv && uv sync --no-dev -COPY requirements.txt ./ -RUN uv pip install -r requirements.txt -COPY ../../../ . -RUN mkdir -p /workspace -CMD ["uv", "run", "python", "-m", "src.toolbox.workflows.dependency_analysis.workflow"] -``` +**How it works:** +1. Your workflow code lives in `backend/toolbox/workflows/{workflow_name}/` +2. This directory is **mounted as a volume** in the worker container at `/app/toolbox/workflows/` +3. Worker discovers and registers your workflow automatically on startup +4. When submitted, the workflow runs inside the long-lived worker container + +**Benefits:** +- Zero container build time per workflow +- Instant code changes (just restart worker) +- All toolchains pre-installed (AFL++, cargo-fuzz, apktool, etc.) +- Consistent environment across all workflows of the same vertical --- -## Step 6: Register and Test Your Workflow +## Step 6: Test Your Workflow -- Add your workflow to the registry (e.g., `backend/toolbox/workflows/registry.py`) -- Write a test script or use the CLI to submit a workflow run -- Check that SARIF results are produced and stored as expected +### Using the CLI -Example test: +```bash +# Start FuzzForge with Temporal +docker-compose -f docker-compose.temporal.yaml up -d + +# Wait for services to initialize +sleep 10 + +# Submit workflow with file upload +cd test_projects/vulnerable_app/ +fuzzforge workflow run dependency_analysis . + +# CLI automatically: +# - Creates tarball of current directory +# - Uploads to MinIO via backend +# - Submits workflow with target_id +# - Worker downloads from MinIO and executes +``` + +### Using Python SDK ```python -import asyncio -from backend.src.toolbox.workflows.dependency_analysis.workflow import main_flow +from fuzzforge_sdk import FuzzForgeClient +from pathlib import Path -async def test_workflow(): - result = await main_flow(target_path="/tmp/test-project", scan_dev_dependencies=True) - print(result) +client = FuzzForgeClient(base_url="http://localhost:8000") -if __name__ == "__main__": - asyncio.run(test_workflow()) +# Submit with automatic upload +response = client.submit_workflow_with_upload( + workflow_name="dependency_analysis", + target_path=Path("/path/to/project"), + parameters={ + "scan_dev_dependencies": True, + "vulnerability_threshold": "medium" + } +) + +print(f"Workflow started: {response.run_id}") + +# Wait for completion +final_status = client.wait_for_completion(response.run_id) + +# Get findings +findings = client.get_run_findings(response.run_id) +print(findings.sarif) + +client.close() ``` +### Check Temporal UI + +Open http://localhost:8233 to see: +- Workflow execution timeline +- Activity results +- Logs and errors +- Retry history + --- ## Best Practices diff --git a/docs/docs/how-to/debugging.md b/docs/docs/how-to/debugging.md new file mode 100644 index 0000000..fd94e73 --- /dev/null +++ b/docs/docs/how-to/debugging.md @@ -0,0 +1,453 @@ +# Debugging Workflows and Modules + +This guide shows you how to debug FuzzForge workflows and modules using Temporal's powerful debugging features. + +--- + +## Quick Debugging Checklist + +When something goes wrong: + +1. **Check worker logs** - `docker-compose -f docker-compose.temporal.yaml logs worker-rust -f` +2. **Check Temporal UI** - http://localhost:8233 (visual execution history) +3. **Check MinIO console** - http://localhost:9001 (inspect uploaded files) +4. **Check backend logs** - `docker-compose -f docker-compose.temporal.yaml logs fuzzforge-backend -f` + +--- + +## Debugging Workflow Discovery + +### Problem: Workflow Not Found + +**Symptom:** Worker logs show "No workflows found for vertical: rust" + +**Debug Steps:** + +1. **Check if worker can see the workflow:** + ```bash + docker exec fuzzforge-worker-rust ls /app/toolbox/workflows/ + ``` + +2. **Check metadata.yaml exists:** + ```bash + docker exec fuzzforge-worker-rust cat /app/toolbox/workflows/my_workflow/metadata.yaml + ``` + +3. **Verify vertical field matches:** + ```bash + docker exec fuzzforge-worker-rust grep "vertical:" /app/toolbox/workflows/my_workflow/metadata.yaml + ``` + Should output: `vertical: rust` + +4. **Check worker logs for discovery errors:** + ```bash + docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "my_workflow" + ``` + +**Solution:** +- Ensure `metadata.yaml` has correct `vertical` field +- Restart worker to reload: `docker-compose -f docker-compose.temporal.yaml restart worker-rust` +- Check worker logs for discovery confirmation + +--- + +## Debugging Workflow Execution + +### Using Temporal Web UI + +The Temporal UI at http://localhost:8233 is your primary debugging tool. + +**Navigate to a workflow:** +1. Open http://localhost:8233 +2. Click "Workflows" in left sidebar +3. Find your workflow by `run_id` or workflow name +4. Click to see detailed execution + +**What you can see:** +- **Execution timeline** - When each activity started/completed +- **Input/output** - Exact parameters passed to workflow +- **Activity results** - Return values from each activity +- **Error stack traces** - Full Python tracebacks +- **Retry history** - All retry attempts with reasons +- **Worker information** - Which worker executed each activity + +**Example: Finding why an activity failed:** +1. Open workflow in Temporal UI +2. Scroll to failed activity (marked in red) +3. Click on the activity +4. See full error message and stack trace +5. Check "Input" tab to see what parameters were passed + +--- + +## Viewing Worker Logs + +### Real-time Monitoring + +```bash +# Follow logs from rust worker +docker-compose -f docker-compose.temporal.yaml logs worker-rust -f + +# Follow logs from all workers +docker-compose -f docker-compose.temporal.yaml logs worker-rust worker-android -f + +# Show last 100 lines +docker-compose -f docker-compose.temporal.yaml logs worker-rust --tail 100 +``` + +### What Worker Logs Show + +**On startup:** +``` +INFO: Scanning for workflows in: /app/toolbox/workflows +INFO: Importing workflow module: toolbox.workflows.security_assessment.workflow +INFO: āœ“ Discovered workflow: SecurityAssessmentWorkflow from security_assessment (vertical: rust) +INFO: šŸš€ Worker started for vertical 'rust' +``` + +**During execution:** +``` +INFO: Starting SecurityAssessmentWorkflow (workflow_id=security_assessment-abc123, target_id=548193a1...) +INFO: Downloading target from MinIO: 548193a1-f73f-4ec1-8068-19ec2660b8e4 +INFO: Executing activity: scan_files +INFO: Completed activity: scan_files (duration: 3.2s) +``` + +**On errors:** +``` +ERROR: Failed to import workflow module toolbox.workflows.broken.workflow: + File "/app/toolbox/workflows/broken/workflow.py", line 42 + def run( +IndentationError: expected an indented block +``` + +### Filtering Logs + +```bash +# Show only errors +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep ERROR + +# Show workflow discovery +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "Discovered workflow" + +# Show specific workflow execution +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "security_assessment-abc123" + +# Show activity execution +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep "activity" +``` + +--- + +## Debugging File Upload + +### Check if File Was Uploaded + +**Using MinIO Console:** +1. Open http://localhost:9001 +2. Login: `fuzzforge` / `fuzzforge123` +3. Click "Buckets" ā†’ "targets" +4. Look for your `target_id` (UUID format) +5. Click to download and inspect locally + +**Using CLI:** +```bash +# Check MinIO status +curl http://localhost:9000 + +# List backend logs for upload +docker-compose -f docker-compose.temporal.yaml logs fuzzforge-backend | grep "upload" +``` + +### Check Worker Cache + +```bash +# List cached targets +docker exec fuzzforge-worker-rust ls -lh /cache/ + +# Check specific target +docker exec fuzzforge-worker-rust ls -lh /cache/548193a1-f73f-4ec1-8068-19ec2660b8e4 +``` + +--- + +## Interactive Debugging + +### Access Running Worker + +```bash +# Open shell in worker container +docker exec -it fuzzforge-worker-rust bash + +# Now you can: +# - Check filesystem +ls -la /app/toolbox/workflows/ + +# - Test imports +python3 -c "from toolbox.workflows.my_workflow.workflow import MyWorkflow; print(MyWorkflow)" + +# - Check environment variables +env | grep TEMPORAL + +# - Test activities +cd /app/toolbox/workflows/my_workflow +python3 -c "from activities import my_activity; print(my_activity)" + +# - Check cache +ls -lh /cache/ +``` + +### Test Module in Isolation + +```bash +# Enter worker container +docker exec -it fuzzforge-worker-rust bash + +# Navigate to module +cd /app/toolbox/modules/scanner + +# Run module directly +python3 -c " +from file_scanner import FileScannerModule +scanner = FileScannerModule() +print(scanner.get_metadata()) +" +``` + +--- + +## Debugging Module Code + +### Edit and Reload + +Since toolbox is mounted as a volume, you can edit code on your host and reload: + +1. **Edit module on host:** + ```bash + # On your host machine + vim backend/toolbox/modules/scanner/file_scanner.py + ``` + +2. **Restart worker to reload:** + ```bash + docker-compose -f docker-compose.temporal.yaml restart worker-rust + ``` + +3. **Check discovery logs:** + ```bash + docker-compose -f docker-compose.temporal.yaml logs worker-rust | tail -50 + ``` + +### Add Debug Logging + +Add logging to your workflow or module: + +```python +import logging + +logger = logging.getLogger(__name__) + +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self, target_id: str): + workflow.logger.info(f"Starting with target_id: {target_id}") # Shows in Temporal UI + + logger.info("Processing step 1") # Shows in worker logs + logger.debug(f"Debug info: {some_variable}") # Shows if LOG_LEVEL=DEBUG + + try: + result = await some_activity() + logger.info(f"Activity result: {result}") + except Exception as e: + logger.error(f"Activity failed: {e}", exc_info=True) # Full stack trace + raise +``` + +Set debug logging: +```bash +# Edit docker-compose.temporal.yaml +services: + worker-rust: + environment: + LOG_LEVEL: DEBUG # Change from INFO to DEBUG + +# Restart +docker-compose -f docker-compose.temporal.yaml restart worker-rust +``` + +--- + +## Common Issues and Solutions + +### Issue: Workflow stuck in "Running" state + +**Debug:** +1. Check Temporal UI for last completed activity +2. Check worker logs for errors +3. Check if worker is still running: `docker-compose -f docker-compose.temporal.yaml ps worker-rust` + +**Solution:** +- Worker may have crashed - restart it +- Activity may be hanging - check for infinite loops or stuck network calls +- Check worker resource limits: `docker stats fuzzforge-worker-rust` + +### Issue: Import errors in workflow + +**Debug:** +1. Check worker logs for full error trace +2. Check if module file exists: + ```bash + docker exec fuzzforge-worker-rust ls /app/toolbox/modules/my_module/ + ``` + +**Solution:** +- Ensure module is in correct directory +- Check for syntax errors: `docker exec fuzzforge-worker-rust python3 -m py_compile /app/toolbox/modules/my_module/my_module.py` +- Verify imports are correct + +### Issue: Target file not found in worker + +**Debug:** +1. Check if target exists in MinIO console +2. Check worker logs for download errors +3. Verify target_id is correct + +**Solution:** +- Re-upload file via CLI +- Check MinIO is running: `docker-compose -f docker-compose.temporal.yaml ps minio` +- Check MinIO credentials in worker environment + +--- + +## Performance Debugging + +### Check Activity Duration + +**In Temporal UI:** +1. Open workflow execution +2. Scroll through activities +3. Each shows duration (e.g., "3.2s") +4. Identify slow activities + +### Monitor Resource Usage + +```bash +# Monitor worker resource usage +docker stats fuzzforge-worker-rust + +# Check worker logs for memory warnings +docker-compose -f docker-compose.temporal.yaml logs worker-rust | grep -i "memory\|oom" +``` + +### Profile Workflow Execution + +Add timing to your workflow: + +```python +import time + +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self, target_id: str): + start = time.time() + + result1 = await activity1() + workflow.logger.info(f"Activity1 took: {time.time() - start:.2f}s") + + start = time.time() + result2 = await activity2() + workflow.logger.info(f"Activity2 took: {time.time() - start:.2f}s") +``` + +--- + +## Advanced Debugging + +### Enable Temporal Worker Debug Logs + +```bash +# Edit docker-compose.temporal.yaml +services: + worker-rust: + environment: + TEMPORAL_LOG_LEVEL: DEBUG + LOG_LEVEL: DEBUG + +# Restart +docker-compose -f docker-compose.temporal.yaml restart worker-rust +``` + +### Inspect Temporal Workflows via CLI + +```bash +# Install Temporal CLI +docker exec fuzzforge-temporal tctl + +# List workflows +docker exec fuzzforge-temporal tctl workflow list + +# Describe workflow +docker exec fuzzforge-temporal tctl workflow describe -w security_assessment-abc123 + +# Show workflow history +docker exec fuzzforge-temporal tctl workflow show -w security_assessment-abc123 +``` + +### Check Network Connectivity + +```bash +# From worker to Temporal +docker exec fuzzforge-worker-rust ping temporal + +# From worker to MinIO +docker exec fuzzforge-worker-rust curl http://minio:9000 + +# From host to services +curl http://localhost:8233 # Temporal UI +curl http://localhost:9000 # MinIO +curl http://localhost:8000/health # Backend +``` + +--- + +## Debugging Best Practices + +1. **Always check Temporal UI first** - It shows the most complete execution history +2. **Use structured logging** - Include workflow_id, target_id in log messages +3. **Log at decision points** - Before/after each major operation +4. **Keep worker logs** - They persist across workflow runs +5. **Test modules in isolation** - Use `docker exec` to test before integrating +6. **Use debug builds** - Enable DEBUG logging during development +7. **Monitor resources** - Use `docker stats` to catch resource issues + +--- + +## Getting Help + +If you're still stuck: + +1. **Collect diagnostic info:** + ```bash + # Save all logs + docker-compose -f docker-compose.temporal.yaml logs > fuzzforge-logs.txt + + # Check service status + docker-compose -f docker-compose.temporal.yaml ps > service-status.txt + ``` + +2. **Check Temporal UI** and take screenshots of: + - Workflow execution timeline + - Failed activity details + - Error messages + +3. **Report issue** with: + - Workflow name and run_id + - Error messages from logs + - Screenshots from Temporal UI + - Steps to reproduce + +--- + +**Happy debugging!** šŸ›šŸ” diff --git a/docs/docs/how-to/docker-setup.md b/docs/docs/how-to/docker-setup.md index 1785f74..6448726 100644 --- a/docs/docs/how-to/docker-setup.md +++ b/docs/docs/how-to/docker-setup.md @@ -97,6 +97,43 @@ If you prefer, you can use a systemd override to add the registry flag. See the --- +## Worker Profiles (Resource Optimization - v0.7.0) + +FuzzForge workers use Docker Compose profiles to prevent auto-startup: + +```yaml +# docker-compose.yml +worker-ossfuzz: + profiles: + - workers # For starting all workers + - ossfuzz # For starting just this worker + restart: "no" # Don't auto-restart +``` + +### Behavior + +- **`docker-compose up -d`**: Workers DON'T start (saves ~6-7GB RAM) +- **CLI workflows**: Workers start automatically on-demand +- **Manual start**: `docker start fuzzforge-worker-ossfuzz` + +### Resource Savings + +| Command | Workers Started | RAM Usage | +|---------|----------------|-----------| +| `docker-compose up -d` | None (core only) | ~1.2 GB | +| `ff workflow run ossfuzz_campaign .` | ossfuzz worker only | ~3-5 GB | +| `docker-compose --profile workers up -d` | All workers | ~8 GB | + +### Starting All Workers (Legacy Behavior) + +If you prefer the old behavior where all workers start: + +```bash +docker-compose --profile workers up -d +``` + +--- + ## Common Issues & How to Fix Them ### "x509: certificate signed by unknown authority" diff --git a/docs/docs/how-to/mcp-integration.md b/docs/docs/how-to/mcp-integration.md index 79a5506..3470bd9 100644 --- a/docs/docs/how-to/mcp-integration.md +++ b/docs/docs/how-to/mcp-integration.md @@ -83,7 +83,6 @@ You should see status responses and endpoint listings. "parameters": { "workflow_name": "infrastructure_scan", "target_path": "/path/to/your/project", - "volume_mode": "ro", "parameters": { "checkov_config": { "severity": ["HIGH", "MEDIUM", "LOW"] @@ -183,7 +182,7 @@ curl http://localhost:8000/workflows/ ```bash curl -X POST http://localhost:8000/workflows/infrastructure_scan/submit \ -H "Content-Type: application/json" \ - -d '{"target_path": "/your/path", "volume_mode": "ro"}' + -d '{"target_path": "/your/path"}' ``` ### General Support @@ -204,7 +203,7 @@ curl -X POST http://localhost:8000/workflows/infrastructure_scan/submit \ ā”‚ ā”‚ ā–¼ ā–¼ ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” - ā”‚ MCP Tools ā”‚ ā”‚ Prefect ā”‚ + ā”‚ MCP Tools ā”‚ ā”‚ Temporal ā”‚ ā”‚ - scan submit ā”‚ ā”‚ Workflows ā”‚ ā”‚ - results ā”‚ ā”‚ - Security ā”‚ ā”‚ - analysis ā”‚ ā”‚ - Fuzzing ā”‚ diff --git a/docs/docs/how-to/troubleshooting.md b/docs/docs/how-to/troubleshooting.md index 0161919..76cac2b 100644 --- a/docs/docs/how-to/troubleshooting.md +++ b/docs/docs/how-to/troubleshooting.md @@ -10,15 +10,16 @@ Before diving into specific errors, letā€™s check the basics: ```bash # Check all FuzzForge services -docker compose ps +docker-compose -f docker-compose.temporal.yaml ps -# Verify Docker registry config +# Verify Docker registry config (if using workflow registry) docker info | grep -i "insecure registries" # Test service health endpoints curl http://localhost:8000/health -curl http://localhost:4200 -curl http://localhost:5001/v2/ +curl http://localhost:8233 # Temporal Web UI +curl http://localhost:9000 # MinIO API +curl http://localhost:9001 # MinIO Console ``` If any of these commands fail, note the error message and continue below. @@ -51,15 +52,17 @@ Docker is trying to use HTTPS for the local registry, but itā€™s set up for HTTP The registry isnā€™t running or the port is blocked. **How to fix:** -- Make sure the registry container is up: +- Make sure the registry container is up (if using registry for workflow images): ```bash - docker compose ps registry + docker-compose -f docker-compose.temporal.yaml ps registry ``` - Check logs for errors: ```bash - docker compose logs registry + docker-compose -f docker-compose.temporal.yaml logs registry ``` -- If port 5001 is in use, change it in `docker-compose.yaml` and your Docker config. +- If port 5001 is in use, change it in `docker-compose.temporal.yaml` and your Docker config. + +**Note:** With Temporal architecture, target files use MinIO (port 9000), not the registry. ### "no such host" error @@ -74,31 +77,42 @@ Docker canā€™t resolve `localhost`. ## Workflow Execution Issues -### "mounts denied" or volume errors +### Upload fails or file access errors -**Whatā€™s happening?** -Docker canā€™t access the path you provided. +**What's happening?** +File upload to MinIO failed or worker can't download target. **How to fix:** -- Always use absolute paths. -- On Docker Desktop, add your project directory to File Sharing. -- Confirm the path exists and is readable. - -### Workflow status is "Crashed" or "Late" - -**Whatā€™s happening?** -- "Crashed": Usually a registry, path, or tool error. -- "Late": Worker is overloaded or system is slow. - -**How to fix:** -- Check logs for details: +- Check MinIO is running: ```bash - docker compose logs prefect-worker | tail -50 + docker-compose -f docker-compose.temporal.yaml ps minio ``` +- Check MinIO logs: + ```bash + docker-compose -f docker-compose.temporal.yaml logs minio + ``` +- Verify MinIO is accessible: + ```bash + curl http://localhost:9000 + ``` +- Check file size (max 10GB by default). + +### Workflow status is "Failed" or "Running" (stuck) + +**What's happening?** +- "Failed": Usually a target download, storage, or tool error. +- "Running" (stuck): Worker is overloaded, target download failed, or worker crashed. + +**How to fix:** +- Check worker logs for details: + ```bash + docker-compose -f docker-compose.temporal.yaml logs worker-rust | tail -50 + ``` +- Check Temporal Web UI at http://localhost:8233 for detailed execution history - Restart services: ```bash - docker compose down - docker compose up -d + docker-compose -f docker-compose.temporal.yaml down + docker-compose -f docker-compose.temporal.yaml up -d ``` - Reduce the number of concurrent workflows if your system is resource-constrained. @@ -106,22 +120,23 @@ Docker canā€™t access the path you provided. ## Service Connectivity Issues -### Backend (port 8000) or Prefect UI (port 4200) not responding +### Backend (port 8000) or Temporal UI (port 8233) not responding **How to fix:** - Check if the service is running: ```bash - docker compose ps fuzzforge-backend - docker compose ps prefect-server + docker-compose -f docker-compose.temporal.yaml ps fuzzforge-backend + docker-compose -f docker-compose.temporal.yaml ps temporal ``` - View logs for errors: ```bash - docker compose logs fuzzforge-backend --tail 50 - docker compose logs prefect-server --tail 20 + docker-compose -f docker-compose.temporal.yaml logs fuzzforge-backend --tail 50 + docker-compose -f docker-compose.temporal.yaml logs temporal --tail 20 ``` - Restart the affected service: ```bash - docker compose restart fuzzforge-backend + docker-compose -f docker-compose.temporal.yaml restart fuzzforge-backend + docker-compose -f docker-compose.temporal.yaml restart temporal ``` --- @@ -197,13 +212,13 @@ Docker canā€™t access the path you provided. - Check Docker network configuration: ```bash docker network ls - docker network inspect fuzzforge_default + docker network inspect fuzzforge-temporal_default ``` - Recreate the network: ```bash - docker compose down + docker-compose -f docker-compose.temporal.yaml down docker network prune -f - docker compose up -d + docker-compose -f docker-compose.temporal.yaml up -d ``` --- @@ -229,10 +244,10 @@ Docker canā€™t access the path you provided. ### Enable debug logging ```bash -export PREFECT_LOGGING_LEVEL=DEBUG -docker compose down -docker compose up -d -docker compose logs fuzzforge-backend -f +export TEMPORAL_LOGGING_LEVEL=DEBUG +docker-compose -f docker-compose.temporal.yaml down +docker-compose -f docker-compose.temporal.yaml up -d +docker-compose -f docker-compose.temporal.yaml logs fuzzforge-backend -f ``` ### Collect diagnostic info @@ -243,12 +258,12 @@ Save and run this script to gather info for support: #!/bin/bash echo "=== FuzzForge Diagnostics ===" date -docker compose ps +docker-compose -f docker-compose.temporal.yaml ps docker info | grep -A 5 -i "insecure registries" curl -s http://localhost:8000/health || echo "Backend unhealthy" -curl -s http://localhost:4200 >/dev/null && echo "Prefect UI healthy" || echo "Prefect UI unhealthy" -curl -s http://localhost:5001/v2/ >/dev/null && echo "Registry healthy" || echo "Registry unhealthy" -docker compose logs --tail 10 +curl -s http://localhost:8233 >/dev/null && echo "Temporal UI healthy" || echo "Temporal UI unhealthy" +curl -s http://localhost:9000 >/dev/null && echo "MinIO healthy" || echo "MinIO unhealthy" +docker-compose -f docker-compose.temporal.yaml logs --tail 10 ``` ### Still stuck? diff --git a/docs/docs/reference/cli-ai.md b/docs/docs/reference/cli-ai.md index d7d5c93..a821235 100644 --- a/docs/docs/reference/cli-ai.md +++ b/docs/docs/reference/cli-ai.md @@ -136,7 +136,6 @@ FuzzForge supports the Model Context Protocol (MCP), allowing LLM clients and AI "parameters": { "workflow_name": "infrastructure_scan", "target_path": "/path/to/your/project", - "volume_mode": "ro", "parameters": { "checkov_config": { "severity": ["HIGH", "MEDIUM", "LOW"] @@ -193,7 +192,7 @@ FuzzForge supports the Model Context Protocol (MCP), allowing LLM clients and AI `curl http://localhost:8000/workflows/` - **Scan Submission Errors:** - `curl -X POST http://localhost:8000/workflows/infrastructure_scan/submit -H "Content-Type: application/json" -d '{"target_path": "/your/path", "volume_mode": "ro"}'` + `curl -X POST http://localhost:8000/workflows/infrastructure_scan/submit -H "Content-Type: application/json" -d '{"target_path": "/your/path"}'` - **General Support:** - Check Docker Compose logs: `docker compose logs fuzzforge-backend` diff --git a/docs/docs/tutorial/getting-started.md b/docs/docs/tutorial/getting-started.md index d501735..426b1c6 100644 --- a/docs/docs/tutorial/getting-started.md +++ b/docs/docs/tutorial/getting-started.md @@ -85,24 +85,23 @@ docker pull localhost:5001/hello-world 2>/dev/null || echo "Registry not accessi Start all FuzzForge services: ```bash -docker compose up -d +docker-compose -f docker-compose.temporal.yaml up -d ``` -This will start 8 services: -- **prefect-server**: Workflow orchestration server -- **prefect-worker**: Executes workflows in Docker containers +This will start 6+ services: +- **temporal**: Workflow orchestration server (includes embedded PostgreSQL for dev) +- **minio**: S3-compatible storage for uploaded targets and results +- **minio-setup**: One-time setup for MinIO buckets (exits after setup) - **fuzzforge-backend**: FastAPI backend and workflow management -- **postgres**: Metadata and workflow state storage -- **redis**: Message broker and caching -- **registry**: Local Docker registry for workflow images -- **docker-proxy**: Secure Docker socket proxy -- **prefect-services**: Additional Prefect services +- **worker-rust**: Long-lived worker for Rust/native security analysis +- **worker-android**: Long-lived worker for Android security analysis (if configured) +- **worker-web**: Long-lived worker for web security analysis (if configured) Wait for all services to be healthy (this may take 2-3 minutes on first startup): ```bash # Check service health -docker compose ps +docker-compose -f docker-compose.temporal.yaml ps # Verify FuzzForge is ready curl http://localhost:8000/health @@ -154,33 +153,40 @@ You should see 6 production workflows: ## Step 6: Run Your First Workflow -Let's run a static analysis workflow on one of the included vulnerable test projects. +Let's run a security assessment workflow on one of the included vulnerable test projects. ### Using the CLI (Recommended): ```bash # Navigate to a test project -cd /path/to/fuzzforge/test_projects/static_analysis_vulnerable +cd /path/to/fuzzforge/test_projects/vulnerable_app -# Submit the workflow -fuzzforge runs submit static_analysis_scan . +# Submit the workflow - CLI automatically uploads the local directory +fuzzforge workflow run security_assessment . + +# The CLI will: +# 1. Detect that '.' is a local directory +# 2. Create a compressed tarball of the directory +# 3. Upload it to the backend via HTTP +# 4. The backend stores it in MinIO +# 5. The worker downloads it when ready to analyze # Monitor the workflow -fuzzforge runs status +fuzzforge workflow status # View results when complete -fuzzforge findings get +fuzzforge finding ``` ### Using the API: +For local files, you can use the upload endpoint: + ```bash -# Submit workflow -curl -X POST "http://localhost:8000/workflows/static_analysis_scan/submit" \ - -H "Content-Type: application/json" \ - -d '{ - "target_path": "/path/to/your/project" - }' +# Create tarball and upload +tar -czf project.tar.gz /path/to/your/project +curl -X POST "http://localhost:8000/workflows/security_assessment/upload-and-submit" \ + -F "file=@project.tar.gz" # Check status curl "http://localhost:8000/runs/{run-id}/status" @@ -189,6 +195,8 @@ curl "http://localhost:8000/runs/{run-id}/status" curl "http://localhost:8000/runs/{run-id}/findings" ``` +**Note**: The CLI handles file upload automatically. For remote workflows where the target path exists on the backend server, you can still use path-based submission for backward compatibility. + ## Step 7: Understanding the Results The workflow will complete in 30-60 seconds and return results in SARIF format. For the test project, you should see: @@ -216,13 +224,19 @@ Example output: } ``` -## Step 8: Access the Prefect Dashboard +## Step 8: Access the Temporal Web UI -You can monitor workflow execution in real-time using the Prefect dashboard: +You can monitor workflow execution in real-time using the Temporal Web UI: -1. Open http://localhost:4200 in your browser -2. Navigate to "Flow Runs" to see workflow executions -3. Click on a run to see detailed logs and execution graph +1. Open http://localhost:8233 in your browser +2. Navigate to "Workflows" to see workflow executions +3. Click on a workflow to see detailed execution history and activity results + +You can also access the MinIO console to view uploaded targets: + +1. Open http://localhost:9001 in your browser +2. Login with: `fuzzforge` / `fuzzforge123` +3. Browse the `targets` bucket to see uploaded files ## Next Steps @@ -242,9 +256,10 @@ Congratulations! You've successfully: If you encounter problems: 1. **Workflow crashes with registry errors**: Check Docker insecure registry configuration -2. **Services won't start**: Ensure ports 4200, 5001, 8000 are available +2. **Services won't start**: Ensure ports 8000, 8233, 9000, 9001 are available 3. **No findings returned**: Verify the target path contains analyzable code files 4. **CLI not found**: Ensure Python/pip installation path is in your PATH +5. **Upload fails**: Check that MinIO is running and accessible at http://localhost:9000 See the [Troubleshooting Guide](../how-to/troubleshooting.md) for detailed solutions. diff --git a/docs/index.md b/docs/index.md index 8e30cb8..fc2a6a8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # FuzzForge Documentation -Welcome to FuzzForge, a comprehensive security analysis platform built on Prefect 3 that automates security testing workflows. FuzzForge provides 6 production-ready workflows that run static analysis, secret detection, infrastructure scanning, penetration testing, and custom fuzzing campaigns with Docker-based isolation and SARIF-compliant reporting. +Welcome to FuzzForge, a comprehensive security analysis platform built on Temporal that automates security testing workflows. FuzzForge provides production-ready workflows that run static analysis, secret detection, infrastructure scanning, penetration testing, and custom fuzzing campaigns with Docker-based isolation and SARIF-compliant reporting. ## šŸš€ Quick Navigation diff --git a/examples/test_a2a_simple.py b/examples/test_a2a_simple.py new file mode 100644 index 0000000..5b9cd9e --- /dev/null +++ b/examples/test_a2a_simple.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +""" +Simple example of using the A2A wrapper +Run from project root: python examples/test_a2a_simple.py +""" +import asyncio + + +async def main(): + # Clean import! + from fuzzforge_ai.a2a_wrapper import send_agent_task + + print("Sending task to agent at http://127.0.0.1:10900...") + + result = await send_agent_task( + url="http://127.0.0.1:10900/a2a/litellm_agent", + model="gpt-4o-mini", + provider="openai", + prompt="You are concise.", + message="Give me a simple Python function that adds two numbers.", + context="test_session", + timeout=120 + ) + + print(f"\nContext ID: {result.context_id}") + print(f"\nResponse:\n{result.text}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/ci-start.sh b/scripts/ci-start.sh new file mode 100755 index 0000000..368fee6 --- /dev/null +++ b/scripts/ci-start.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# FuzzForge CI/CD Startup Script +# This script configures Docker and starts FuzzForge services for CI/CD environments +set -e + +echo "šŸš€ Starting FuzzForge for CI/CD..." + +# Configure Docker for insecure registry (required for localhost:5001) +echo "šŸ“ Configuring Docker for local registry..." +if [ -f /etc/docker/daemon.json ]; then + # Merge with existing config if jq is available + if command -v jq &> /dev/null; then + echo " Merging with existing Docker config..." + jq '. + {"insecure-registries": (."insecure-registries" // []) + ["localhost:5001"] | unique}' \ + /etc/docker/daemon.json > /tmp/daemon.json + sudo mv /tmp/daemon.json /etc/docker/daemon.json + else + echo " āš ļø jq not found, overwriting Docker config (backup created)" + sudo cp /etc/docker/daemon.json /etc/docker/daemon.json.backup + echo '{"insecure-registries": ["localhost:5001"]}' | sudo tee /etc/docker/daemon.json > /dev/null + fi +else + echo " Creating new Docker config..." + echo '{"insecure-registries": ["localhost:5001"]}' | sudo tee /etc/docker/daemon.json > /dev/null +fi + +# Restart Docker daemon +echo "šŸ”„ Restarting Docker daemon..." +if command -v systemctl &> /dev/null; then + sudo systemctl restart docker +else + sudo service docker restart +fi + +# Wait for Docker to be ready +echo "ā³ Waiting for Docker to be ready..." +timeout 30 bash -c 'until docker ps &> /dev/null; do sleep 1; done' || { + echo "āŒ Docker failed to start" + exit 1 +} +echo " āœ“ Docker is ready" + +# Start FuzzForge services +echo "" +echo "šŸ³ Starting FuzzForge services (core only, workers on-demand)..." +echo " This will start:" +echo " ā€¢ Temporal (workflow engine)" +echo " ā€¢ PostgreSQL (Temporal database)" +echo " ā€¢ MinIO (object storage)" +echo " ā€¢ Backend (API server)" +echo "" + +# Check if docker-compose or docker compose is available +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null; then + COMPOSE_CMD="docker compose" +else + echo "āŒ docker-compose not found" + exit 1 +fi + +# Start services +$COMPOSE_CMD up -d + +# Wait for backend health +echo "" +echo "ā³ Waiting for services to be healthy (up to 2 minutes)..." +echo " Checking backend health at http://localhost:8000/health" +SECONDS=0 +timeout 120 bash -c 'until curl -sf http://localhost:8000/health > /dev/null 2>&1; do + if [ $((SECONDS % 10)) -eq 0 ]; then + echo " Still waiting... (${SECONDS}s elapsed)" + fi + sleep 3 +done' || { + echo "" + echo "āŒ Services failed to become healthy within 2 minutes" + echo "" + echo "Troubleshooting:" + echo " ā€¢ Check logs: docker-compose logs" + echo " ā€¢ Check status: docker-compose ps" + echo " ā€¢ Check backend logs: docker logs fuzzforge-backend" + exit 1 +} + +echo "" +echo "āœ… FuzzForge is ready! (startup took ${SECONDS}s)" +echo "" +echo "šŸ“Š Service Status:" +$COMPOSE_CMD ps + +echo "" +echo "šŸŽÆ Next steps:" +echo " 1. Initialize FuzzForge project:" +echo " ff init --api-url http://localhost:8000" +echo "" +echo " 2. Run a security scan:" +echo " ff workflow run security_assessment . --wait --fail-on critical,high" +echo "" +echo " 3. Export results:" +echo " ff workflow run security_assessment . --wait --export-sarif results.sarif" +echo "" diff --git a/scripts/ci-stop.sh b/scripts/ci-stop.sh new file mode 100755 index 0000000..eebe962 --- /dev/null +++ b/scripts/ci-stop.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# FuzzForge CI/CD Cleanup Script +# This script stops and cleans up FuzzForge services after CI/CD execution +set -e + +echo "šŸ›‘ Stopping FuzzForge services..." + +# Check if docker-compose or docker compose is available +if command -v docker-compose &> /dev/null; then + COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null; then + COMPOSE_CMD="docker compose" +else + echo "āš ļø docker-compose not found, skipping cleanup" + exit 0 +fi + +# Stop and remove containers, networks, and volumes +echo " Stopping containers..." +$COMPOSE_CMD down -v --remove-orphans + +echo "" +echo "āœ… FuzzForge stopped and cleaned up" +echo "" +echo "šŸ“Š Resources freed:" +echo " ā€¢ All containers removed" +echo " ā€¢ All volumes removed" +echo " ā€¢ All networks removed" +echo "" diff --git a/sdk/README.md b/sdk/README.md index 5e60e42..ea6a34f 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -5,6 +5,7 @@ A comprehensive Python SDK for the FuzzForge security testing workflow orchestra ## Features - **Complete API Coverage**: All FuzzForge API endpoints supported +- **File Upload**: Automatic tarball creation and multipart upload for local files - **Async & Sync**: Both synchronous and asynchronous client methods - **Real-time Monitoring**: WebSocket and Server-Sent Events for live fuzzing updates - **Type Safety**: Full Pydantic model validation for all data structures @@ -27,9 +28,11 @@ pip install fuzzforge-sdk ## Quick Start +### Method 1: File Upload (Recommended) + ```python from fuzzforge_sdk import FuzzForgeClient -from fuzzforge_sdk.utils import create_workflow_submission +from pathlib import Path # Initialize client client = FuzzForgeClient(base_url="http://localhost:8000") @@ -37,14 +40,19 @@ client = FuzzForgeClient(base_url="http://localhost:8000") # List available workflows workflows = client.list_workflows() -# Submit a workflow -submission = create_workflow_submission( - target_path="/path/to/your/project", - volume_mode="ro", +# Submit a workflow with automatic file upload +target_path = Path("/path/to/your/project") +response = client.submit_workflow_with_upload( + workflow_name="security_assessment", + target_path=target_path, timeout=300 ) -response = client.submit_workflow("static-analysis", submission) +# The SDK automatically: +# - Creates a tarball if target_path is a directory +# - Uploads the file to the backend via HTTP +# - Backend stores it in MinIO +# - Returns the workflow run_id # Wait for completion and get results final_status = client.wait_for_completion(response.run_id) @@ -53,6 +61,26 @@ findings = client.get_run_findings(response.run_id) client.close() ``` +### Method 2: Path-Based Submission (Legacy) + +```python +from fuzzforge_sdk import FuzzForgeClient +from fuzzforge_sdk.utils import create_workflow_submission + +# Initialize client +client = FuzzForgeClient(base_url="http://localhost:8000") + +# Submit a workflow with path (only works if backend can access the path) +submission = create_workflow_submission( + target_path="/path/on/backend/filesystem", + timeout=300 +) + +response = client.submit_workflow("security_assessment", submission) + +client.close() +``` + ## Examples The `examples/` directory contains complete working examples: @@ -61,6 +89,181 @@ The `examples/` directory contains complete working examples: - **`fuzzing_monitor.py`**: Real-time fuzzing monitoring with WebSocket/SSE - **`batch_analysis.py`**: Batch analysis of multiple projects +## File Upload API Reference + +### `submit_workflow_with_upload()` + +Submit a workflow with automatic file upload from local filesystem. + +```python +def submit_workflow_with_upload( + self, + workflow_name: str, + target_path: Union[str, Path], + parameters: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = None, + progress_callback: Optional[Callable[[int, int], None]] = None +) -> RunSubmissionResponse: + """ + Submit workflow with file upload. + + Args: + workflow_name: Name of the workflow to execute + target_path: Path to file or directory to upload + parameters: Optional workflow parameters + timeout: Optional execution timeout in seconds + progress_callback: Optional callback(bytes_sent, total_bytes) + + Returns: + RunSubmissionResponse with run_id and status + + Raises: + FileNotFoundError: If target_path doesn't exist + ValidationError: If parameters are invalid + FuzzForgeHTTPError: If upload fails + """ +``` + +**Example with progress tracking:** + +```python +from fuzzforge_sdk import FuzzForgeClient +from pathlib import Path + +def upload_progress(bytes_sent, total_bytes): + pct = (bytes_sent / total_bytes) * 100 + print(f"Upload progress: {pct:.1f}% ({bytes_sent}/{total_bytes} bytes)") + +client = FuzzForgeClient(base_url="http://localhost:8000") + +response = client.submit_workflow_with_upload( + workflow_name="security_assessment", + target_path=Path("./my-project"), + parameters={"check_secrets": True}, + progress_callback=upload_progress +) + +print(f"Workflow started: {response.run_id}") +``` + +### `asubmit_workflow_with_upload()` + +Async version of `submit_workflow_with_upload()`. + +```python +import asyncio +from fuzzforge_sdk import FuzzForgeClient + +async def main(): + client = FuzzForgeClient(base_url="http://localhost:8000") + + response = await client.asubmit_workflow_with_upload( + workflow_name="security_assessment", + target_path="/path/to/project", + parameters={"timeout": 3600} + ) + + print(f"Workflow started: {response.run_id}") + await client.aclose() + +asyncio.run(main()) +``` + +### Internal: `_create_tarball()` + +Creates a compressed tarball from a file or directory. + +```python +def _create_tarball( + self, + source_path: Path, + progress_callback: Optional[Callable[[int], None]] = None +) -> Path: + """ + Create compressed tarball (.tar.gz) from source. + + Args: + source_path: Path to file or directory + progress_callback: Optional callback(files_added) + + Returns: + Path to created tarball in temp directory + + Note: + Caller is responsible for cleaning up the tarball + """ +``` + +**How it works:** + +1. **Directory**: Creates tarball with all files, preserving structure + ```python + # For directory: /path/to/project/ + # Creates: /tmp/tmpXXXXXX.tar.gz containing: + # project/file1.py + # project/subdir/file2.py + ``` + +2. **Single file**: Creates tarball with just that file + ```python + # For file: /path/to/binary.elf + # Creates: /tmp/tmpXXXXXX.tar.gz containing: + # binary.elf + ``` + +### Upload Flow Diagram + +``` +User Code + ā†“ +submit_workflow_with_upload() + ā†“ +_create_tarball() ā”€ā”€ā”€ā†’ Compress files + ā†“ +HTTP POST multipart/form-data + ā†“ +Backend API (/workflows/{name}/upload-and-submit) + ā†“ +MinIO Storage (S3) ā”€ā”€ā”€ā†’ Store with target_id + ā†“ +Temporal Workflow + ā†“ +Worker downloads from MinIO + ā†“ +Workflow execution +``` + +### Error Handling + +The SDK provides detailed error context: + +```python +from fuzzforge_sdk import FuzzForgeClient +from fuzzforge_sdk.exceptions import ( + FuzzForgeHTTPError, + ValidationError, + ConnectionError +) + +client = FuzzForgeClient(base_url="http://localhost:8000") + +try: + response = client.submit_workflow_with_upload( + workflow_name="security_assessment", + target_path="./nonexistent", + ) +except FileNotFoundError as e: + print(f"Target not found: {e}") +except ValidationError as e: + print(f"Invalid parameters: {e}") +except FuzzForgeHTTPError as e: + print(f"Upload failed (HTTP {e.status_code}): {e.message}") + if e.context.response_data: + print(f"Server response: {e.context.response_data}") +except ConnectionError as e: + print(f"Cannot connect to backend: {e}") +``` + ## Development Install with development dependencies: diff --git a/sdk/examples/basic_workflow.py b/sdk/examples/basic_workflow.py index ec1fd09..74b3a49 100644 --- a/sdk/examples/basic_workflow.py +++ b/sdk/examples/basic_workflow.py @@ -25,7 +25,7 @@ import asyncio import time from pathlib import Path -from fuzzforge_sdk import FuzzForgeClient, WorkflowSubmission +from fuzzforge_sdk import FuzzForgeClient from fuzzforge_sdk.utils import create_workflow_submission, format_sarif_summary, format_duration @@ -61,7 +61,7 @@ def main(): # Get workflow metadata metadata = client.get_workflow_metadata(selected_workflow.name) - print(f"šŸ“ Workflow metadata:") + print("šŸ“ Workflow metadata:") print(f" Author: {metadata.author}") print(f" Required modules: {metadata.required_modules}") print(f" Supported volume modes: {metadata.supported_volume_modes}") @@ -81,7 +81,7 @@ def main(): # Submit the workflow print(f"šŸš€ Submitting workflow '{selected_workflow.name}'...") response = client.submit_workflow(selected_workflow.name, submission) - print(f"āœ… Workflow submitted!") + print("āœ… Workflow submitted!") print(f" Run ID: {response.run_id}") print(f" Status: {response.status}") print() @@ -124,7 +124,7 @@ def main(): # Display metadata if findings.metadata: - print(f"šŸ” Metadata:") + print("šŸ” Metadata:") for key, value in findings.metadata.items(): print(f" {key}: {value}") @@ -180,7 +180,7 @@ def main(): # Additional properties properties = result.get('properties', {}) if properties: - print(f" Properties:") + print(" Properties:") for prop_key, prop_value in properties.items(): print(f" {prop_key}: {prop_value}") diff --git a/sdk/examples/batch_analysis.py b/sdk/examples/batch_analysis.py index a8fa25b..5ac46bc 100644 --- a/sdk/examples/batch_analysis.py +++ b/sdk/examples/batch_analysis.py @@ -27,18 +27,14 @@ from typing import List, Dict, Any import time from fuzzforge_sdk import ( - FuzzForgeClient, - WorkflowSubmission, - WorkflowFindings, - RunSubmissionResponse + FuzzForgeClient ) from fuzzforge_sdk.utils import ( create_workflow_submission, format_sarif_summary, count_sarif_severity_levels, save_sarif_to_file, - get_project_files, - estimate_analysis_time + get_project_files ) @@ -308,7 +304,7 @@ async def main(): batch_duration = batch_end_time - batch_start_time # Generate batch summary report - print(f"\nšŸ“Š Batch Analysis Complete!") + print("\nšŸ“Š Batch Analysis Complete!") print(f" Total time: {batch_duration:.1f}s") print(f" Projects analyzed: {len(analyzer.results)}") @@ -345,7 +341,7 @@ async def main(): print(f" Batch summary: {batch_summary_file}") # Display project summaries - print(f"\nšŸ“ˆ Project Summaries:") + print("\nšŸ“ˆ Project Summaries:") for result in analyzer.results: print(f" {result['project_name']}: " + f"{result['summary']['successful_workflows']}/{result['summary']['total_workflows']} workflows successful, " + diff --git a/sdk/examples/fuzzing_monitor.py b/sdk/examples/fuzzing_monitor.py index 90437df..096574d 100644 --- a/sdk/examples/fuzzing_monitor.py +++ b/sdk/examples/fuzzing_monitor.py @@ -23,11 +23,10 @@ This example demonstrates how to: import asyncio import signal import sys -import time from pathlib import Path from datetime import datetime -from fuzzforge_sdk import FuzzForgeClient, WorkflowSubmission +from fuzzforge_sdk import FuzzForgeClient from fuzzforge_sdk.utils import ( create_workflow_submission, create_resource_limits, @@ -113,7 +112,7 @@ class FuzzingMonitor: corpus_size = stats_data.get('corpus_size', 0) elapsed_time = stats_data.get('elapsed_time', 0) - print(f"šŸ“Š Statistics:") + print("šŸ“Š Statistics:") print(f" Executions: {executions:,}") print(f" Rate: {format_execution_rate(exec_per_sec)}") print(f" Runtime: {format_duration(elapsed_time)}") @@ -123,7 +122,7 @@ class FuzzingMonitor: print(f" Coverage: {coverage:.1f}%") print() - print(f"šŸ’„ Crashes:") + print("šŸ’„ Crashes:") print(f" Total crashes: {crashes}") print(f" Unique crashes: {unique_crashes}") @@ -204,11 +203,11 @@ async def main(): } ) - print(f"šŸš€ Submitting fuzzing workflow...") + print("šŸš€ Submitting fuzzing workflow...") response = await client.asubmit_workflow(selected_workflow.name, submission) monitor.run_id = response.run_id - print(f"āœ… Fuzzing started!") + print("āœ… Fuzzing started!") print(f" Run ID: {response.run_id}") print(f" Initial status: {response.status}") print() diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index e60881a..2afc681 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fuzzforge-sdk" -version = "0.6.0" +version = "0.7.0" description = "Python SDK for FuzzForge security testing workflow orchestration platform" readme = "README.md" authors = [ diff --git a/sdk/src/fuzzforge_sdk/__init__.py b/sdk/src/fuzzforge_sdk/__init__.py index ec7c90c..b0da889 100644 --- a/sdk/src/fuzzforge_sdk/__init__.py +++ b/sdk/src/fuzzforge_sdk/__init__.py @@ -23,8 +23,6 @@ from .models import ( WorkflowListItem, WorkflowStatus, WorkflowFindings, - ResourceLimits, - VolumeMount, FuzzingStats, CrashReport, RunSubmissionResponse, @@ -52,8 +50,6 @@ __all__ = [ "WorkflowListItem", "WorkflowStatus", "WorkflowFindings", - "ResourceLimits", - "VolumeMount", "FuzzingStats", "CrashReport", "RunSubmissionResponse", diff --git a/sdk/src/fuzzforge_sdk/client.py b/sdk/src/fuzzforge_sdk/client.py index 10865bd..1319389 100644 --- a/sdk/src/fuzzforge_sdk/client.py +++ b/sdk/src/fuzzforge_sdk/client.py @@ -19,9 +19,11 @@ including real-time monitoring capabilities for fuzzing workflows. import asyncio import json import logging -from typing import Dict, Any, List, Optional, AsyncIterator, Iterator, Union +import tarfile +import tempfile +from pathlib import Path +from typing import Dict, Any, List, Optional, AsyncIterator, Iterator, Union, Callable from urllib.parse import urljoin, urlparse -import warnings import httpx import websockets @@ -213,6 +215,56 @@ class FuzzForgeClient: response = await self._async_client.get(url) return await self._ahandle_response(response) + def get_workflow_worker_info(self, workflow_name: str) -> Dict[str, Any]: + """ + Get worker information for a workflow. + + Returns details about which worker is required to execute this workflow, + including container name, task queue, and vertical. + + Args: + workflow_name: Name of the workflow + + Returns: + Dictionary with worker info including: + - workflow: Workflow name + - vertical: Worker vertical (e.g., "ossfuzz", "python", "rust") + - worker_container: Docker container name + - task_queue: Temporal task queue name + - required: Whether worker is required (always True) + + Raises: + FuzzForgeHTTPError: If workflow not found or metadata missing + """ + url = urljoin(self.base_url, f"/workflows/{workflow_name}/worker-info") + response = self._client.get(url) + return self._handle_response(response) + + async def aget_workflow_worker_info(self, workflow_name: str) -> Dict[str, Any]: + """ + Get worker information for a workflow (async). + + Returns details about which worker is required to execute this workflow, + including container name, task queue, and vertical. + + Args: + workflow_name: Name of the workflow + + Returns: + Dictionary with worker info including: + - workflow: Workflow name + - vertical: Worker vertical (e.g., "ossfuzz", "python", "rust") + - worker_container: Docker container name + - task_queue: Temporal task queue name + - required: Whether worker is required (always True) + + Raises: + FuzzForgeHTTPError: If workflow not found or metadata missing + """ + url = urljoin(self.base_url, f"/workflows/{workflow_name}/worker-info") + response = await self._async_client.get(url) + return await self._ahandle_response(response) + def submit_workflow( self, workflow_name: str, @@ -235,6 +287,232 @@ class FuzzForgeClient: data = await self._ahandle_response(response) return RunSubmissionResponse(**data) + def _create_tarball( + self, + source_path: Path, + progress_callback: Optional[Callable[[int], None]] = None + ) -> Path: + """ + Create a compressed tarball from a file or directory. + + Args: + source_path: Path to file or directory to archive + progress_callback: Optional callback(bytes_written) for progress tracking + + Returns: + Path to the created tarball + + Raises: + FileNotFoundError: If source_path doesn't exist + """ + if not source_path.exists(): + raise FileNotFoundError(f"Source path not found: {source_path}") + + # Create temp file for tarball + temp_fd, temp_path = tempfile.mkstemp(suffix=".tar.gz") + + try: + logger.info(f"Creating tarball from {source_path}") + + bytes_written = 0 + + with tarfile.open(temp_path, "w:gz") as tar: + if source_path.is_file(): + # Add single file + tar.add(source_path, arcname=source_path.name) + bytes_written = source_path.stat().st_size + if progress_callback: + progress_callback(bytes_written) + else: + # Add directory recursively + for item in source_path.rglob("*"): + if item.is_file(): + arcname = item.relative_to(source_path) + tar.add(item, arcname=arcname) + bytes_written += item.stat().st_size + if progress_callback: + progress_callback(bytes_written) + + tarball_path = Path(temp_path) + tarball_size = tarball_path.stat().st_size + logger.info( + f"Created tarball: {tarball_size / (1024**2):.2f} MB " + f"(compressed from {bytes_written / (1024**2):.2f} MB)" + ) + + return tarball_path + + except Exception: + # Cleanup on error + if Path(temp_path).exists(): + Path(temp_path).unlink() + raise + + def submit_workflow_with_upload( + self, + workflow_name: str, + target_path: Union[str, Path], + parameters: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = None, + progress_callback: Optional[Callable[[int, int], None]] = None + ) -> RunSubmissionResponse: + """ + Submit a workflow with file upload from local filesystem. + + This method automatically creates a tarball if target_path is a directory, + uploads it to the backend, and submits the workflow for execution. + + Args: + workflow_name: Name of the workflow to execute + target_path: Local path to file or directory to analyze + parameters: Workflow-specific parameters + timeout: Timeout in seconds + progress_callback: Optional callback(bytes_uploaded, total_bytes) for progress + + Returns: + Run submission response with run_id + + Raises: + FileNotFoundError: If target_path doesn't exist + FuzzForgeHTTPError: For API errors + """ + target_path = Path(target_path) + tarball_path = None + + try: + # Create tarball if needed + if target_path.is_dir(): + logger.info("Target is directory, creating tarball...") + tarball_path = self._create_tarball(target_path) + upload_file = tarball_path + filename = f"{target_path.name}.tar.gz" + else: + upload_file = target_path + filename = target_path.name + + # Prepare multipart form data + url = urljoin(self.base_url, f"/workflows/{workflow_name}/upload-and-submit") + + files = { + "file": (filename, open(upload_file, "rb"), "application/gzip") + } + + data = {} + + if parameters: + data["parameters"] = json.dumps(parameters) + + if timeout: + data["timeout"] = str(timeout) + + logger.info(f"Uploading {filename} to {workflow_name}...") + + # Track upload progress + if progress_callback: + file_size = upload_file.stat().st_size + + def track_progress(monitor): + progress_callback(monitor.bytes_read, file_size) + + # Note: httpx doesn't have built-in progress tracking for uploads + # This is a placeholder - real implementation would need custom approach + pass + + response = self._client.post(url, files=files, data=data) + + # Close file handle + files["file"][1].close() + + data = self._handle_response(response) + return RunSubmissionResponse(**data) + + finally: + # Cleanup temporary tarball + if tarball_path and tarball_path.exists(): + try: + tarball_path.unlink() + logger.debug(f"Cleaned up temporary tarball: {tarball_path}") + except Exception as e: + logger.warning(f"Failed to cleanup tarball {tarball_path}: {e}") + + async def asubmit_workflow_with_upload( + self, + workflow_name: str, + target_path: Union[str, Path], + parameters: Optional[Dict[str, Any]] = None, + volume_mode: str = "ro", + timeout: Optional[int] = None, + progress_callback: Optional[Callable[[int, int], None]] = None + ) -> RunSubmissionResponse: + """ + Submit a workflow with file upload from local filesystem (async). + + This method automatically creates a tarball if target_path is a directory, + uploads it to the backend, and submits the workflow for execution. + + Args: + workflow_name: Name of the workflow to execute + target_path: Local path to file or directory to analyze + parameters: Workflow-specific parameters + volume_mode: Volume mount mode ("ro" or "rw") + timeout: Timeout in seconds + progress_callback: Optional callback(bytes_uploaded, total_bytes) for progress + + Returns: + Run submission response with run_id + + Raises: + FileNotFoundError: If target_path doesn't exist + FuzzForgeHTTPError: For API errors + """ + target_path = Path(target_path) + tarball_path = None + + try: + # Create tarball if needed + if target_path.is_dir(): + logger.info("Target is directory, creating tarball...") + tarball_path = self._create_tarball(target_path) + upload_file = tarball_path + filename = f"{target_path.name}.tar.gz" + else: + upload_file = target_path + filename = target_path.name + + # Prepare multipart form data + url = urljoin(self.base_url, f"/workflows/{workflow_name}/upload-and-submit") + + files = { + "file": (filename, open(upload_file, "rb"), "application/gzip") + } + + data = {} + + if parameters: + data["parameters"] = json.dumps(parameters) + + if timeout: + data["timeout"] = str(timeout) + + logger.info(f"Uploading {filename} to {workflow_name}...") + + response = await self._async_client.post(url, files=files, data=data) + + # Close file handle + files["file"][1].close() + + response_data = await self._ahandle_response(response) + return RunSubmissionResponse(**response_data) + + finally: + # Cleanup temporary tarball + if tarball_path and tarball_path.exists(): + try: + tarball_path.unlink() + logger.debug(f"Cleaned up temporary tarball: {tarball_path}") + except Exception as e: + logger.warning(f"Failed to cleanup tarball {tarball_path}: {e}") + # Run management methods def get_run_status(self, run_id: str) -> WorkflowStatus: diff --git a/sdk/src/fuzzforge_sdk/docker_logs.py b/sdk/src/fuzzforge_sdk/docker_logs.py deleted file mode 100644 index 960b3f2..0000000 --- a/sdk/src/fuzzforge_sdk/docker_logs.py +++ /dev/null @@ -1,399 +0,0 @@ -""" -Docker log integration for enhanced error reporting. - -This module provides functionality to fetch and parse Docker container logs -to provide better context for deployment and workflow execution errors. -""" -# Copyright (c) 2025 FuzzingLabs -# -# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file -# at the root of this repository for details. -# -# After the Change Date (four years from publication), this version of the -# Licensed Work will be made available under the Apache License, Version 2.0. -# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0 -# -# Additional attribution and requirements are provided in the NOTICE file. - - -import logging -import re -import subprocess -import json -from typing import Dict, Any, List, Optional, Tuple -from datetime import datetime, timezone -from dataclasses import dataclass - -logger = logging.getLogger(__name__) - - -@dataclass -class ContainerLogEntry: - """A single log entry from a container.""" - timestamp: datetime - level: str - message: str - stream: str # 'stdout' or 'stderr' - raw: str - - -@dataclass -class ContainerDiagnostics: - """Complete diagnostics for a container.""" - container_id: Optional[str] - status: str - exit_code: Optional[int] - error: Optional[str] - logs: List[ContainerLogEntry] - resource_usage: Dict[str, Any] - volume_mounts: List[Dict[str, str]] - - -class DockerLogIntegration: - """ - Integration with Docker to fetch container logs and diagnostics. - - This class provides methods to fetch container logs, parse common error - patterns, and extract meaningful diagnostic information from Docker - containers related to FuzzForge workflow execution. - """ - - def __init__(self): - self.docker_available = self._check_docker_availability() - - # Common error patterns in container logs - self.error_patterns = { - 'permission_denied': [ - r'permission denied', - r'operation not permitted', - r'cannot access.*permission denied' - ], - 'out_of_memory': [ - r'out of memory', - r'oom killed', - r'cannot allocate memory' - ], - 'image_pull_failed': [ - r'failed to pull image', - r'pull access denied', - r'image not found' - ], - 'volume_mount_failed': [ - r'invalid mount config', - r'mount denied', - r'no such file or directory.*mount' - ], - 'network_error': [ - r'network is unreachable', - r'connection refused', - r'timeout.*connect' - ], - 'prefect_error': [ - r'prefect.*error', - r'flow run failed', - r'task.*failed' - ] - } - - def _check_docker_availability(self) -> bool: - """Check if Docker is available and accessible.""" - try: - result = subprocess.run(['docker', 'version', '--format', 'json'], - capture_output=True, text=True, timeout=5) - return result.returncode == 0 - except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): - return False - - def get_container_logs(self, container_name_or_id: str, tail: int = 100) -> List[ContainerLogEntry]: - """ - Fetch logs from a Docker container. - - Args: - container_name_or_id: Container name or ID - tail: Number of log lines to retrieve - - Returns: - List of parsed log entries - """ - if not self.docker_available: - logger.warning("Docker not available, cannot fetch container logs") - return [] - - try: - cmd = ['docker', 'logs', '--timestamps', '--tail', str(tail), container_name_or_id] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=10) - - if result.returncode != 0: - logger.error(f"Failed to fetch logs for container {container_name_or_id}: {result.stderr}") - return [] - - return self._parse_docker_logs(result.stdout + result.stderr) - - except subprocess.TimeoutExpired: - logger.error(f"Timeout fetching logs for container {container_name_or_id}") - return [] - except Exception as e: - logger.error(f"Error fetching container logs: {e}") - return [] - - def _parse_docker_logs(self, raw_logs: str) -> List[ContainerLogEntry]: - """Parse raw Docker logs into structured entries.""" - entries = [] - - for line in raw_logs.strip().split('\n'): - if not line.strip(): - continue - - entry = self._parse_log_line(line) - if entry: - entries.append(entry) - - return entries - - def _parse_log_line(self, line: str) -> Optional[ContainerLogEntry]: - """Parse a single log line with timestamp.""" - # Docker log format: 2023-10-01T12:00:00.000000000Z message - timestamp_match = re.match(r'^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+(.*)', line) - - if timestamp_match: - timestamp_str, message = timestamp_match.groups() - try: - timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00')) - except ValueError: - timestamp = datetime.now(timezone.utc) - else: - timestamp = datetime.now(timezone.utc) - message = line - - # Determine log level from message content - level = self._extract_log_level(message) - - # Determine stream (simplified - Docker doesn't clearly separate in combined output) - stream = 'stderr' if any(keyword in message.lower() for keyword in ['error', 'failed', 'exception']) else 'stdout' - - return ContainerLogEntry( - timestamp=timestamp, - level=level, - message=message.strip(), - stream=stream, - raw=line - ) - - def _extract_log_level(self, message: str) -> str: - """Extract log level from message content.""" - message_lower = message.lower() - - if any(keyword in message_lower for keyword in ['error', 'failed', 'exception', 'fatal']): - return 'ERROR' - elif any(keyword in message_lower for keyword in ['warning', 'warn']): - return 'WARNING' - elif any(keyword in message_lower for keyword in ['info', 'information']): - return 'INFO' - elif any(keyword in message_lower for keyword in ['debug']): - return 'DEBUG' - else: - return 'INFO' - - def get_container_diagnostics(self, container_name_or_id: str) -> ContainerDiagnostics: - """ - Get complete diagnostics for a container including logs, status, and resource usage. - - Args: - container_name_or_id: Container name or ID - - Returns: - Complete container diagnostics - """ - if not self.docker_available: - return ContainerDiagnostics( - container_id=None, - status="unknown", - exit_code=None, - error="Docker not available", - logs=[], - resource_usage={}, - volume_mounts=[] - ) - - # Get container inspect data - inspect_data = self._get_container_inspect(container_name_or_id) - - # Get logs - logs = self.get_container_logs(container_name_or_id) - - # Extract key information - if inspect_data: - state = inspect_data.get('State', {}) - config = inspect_data.get('Config', {}) - host_config = inspect_data.get('HostConfig', {}) - - status = state.get('Status', 'unknown') - exit_code = state.get('ExitCode') - error = state.get('Error', '') - - # Get volume mounts - mounts = inspect_data.get('Mounts', []) - volume_mounts = [ - { - 'source': mount.get('Source', ''), - 'destination': mount.get('Destination', ''), - 'mode': mount.get('Mode', ''), - 'type': mount.get('Type', '') - } - for mount in mounts - ] - - # Get resource limits - resource_usage = { - 'memory_limit': host_config.get('Memory', 0), - 'cpu_limit': host_config.get('CpuQuota', 0), - 'cpu_period': host_config.get('CpuPeriod', 0) - } - - else: - status = "not_found" - exit_code = None - error = f"Container {container_name_or_id} not found" - volume_mounts = [] - resource_usage = {} - - return ContainerDiagnostics( - container_id=container_name_or_id, - status=status, - exit_code=exit_code, - error=error, - logs=logs, - resource_usage=resource_usage, - volume_mounts=volume_mounts - ) - - def _get_container_inspect(self, container_name_or_id: str) -> Optional[Dict[str, Any]]: - """Get container inspection data.""" - try: - cmd = ['docker', 'inspect', container_name_or_id] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=5) - - if result.returncode != 0: - return None - - data = json.loads(result.stdout) - return data[0] if data else None - - except (subprocess.TimeoutExpired, json.JSONDecodeError, Exception) as e: - logger.debug(f"Failed to inspect container {container_name_or_id}: {e}") - return None - - def analyze_error_patterns(self, logs: List[ContainerLogEntry]) -> Dict[str, List[str]]: - """ - Analyze logs for common error patterns. - - Args: - logs: List of log entries to analyze - - Returns: - Dictionary mapping error types to matching log messages - """ - detected_errors = {} - - for error_type, patterns in self.error_patterns.items(): - matches = [] - - for log_entry in logs: - for pattern in patterns: - if re.search(pattern, log_entry.message, re.IGNORECASE): - matches.append(log_entry.message) - break # Don't match the same message multiple times - - if matches: - detected_errors[error_type] = matches - - return detected_errors - - def get_container_names_by_label(self, label_filter: str) -> List[str]: - """ - Get container names that match a specific label filter. - - Args: - label_filter: Label filter (e.g., "prefect.flow-run-id=12345") - - Returns: - List of container names - """ - if not self.docker_available: - return [] - - try: - cmd = ['docker', 'ps', '-a', '--filter', f'label={label_filter}', '--format', '{{.Names}}'] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=5) - - if result.returncode != 0: - return [] - - return [name.strip() for name in result.stdout.strip().split('\n') if name.strip()] - - except Exception as e: - logger.debug(f"Failed to get containers by label {label_filter}: {e}") - return [] - - def suggest_fixes(self, error_analysis: Dict[str, List[str]]) -> List[str]: - """ - Suggest fixes based on detected error patterns. - - Args: - error_analysis: Result from analyze_error_patterns() - - Returns: - List of suggested fixes - """ - suggestions = [] - - if 'permission_denied' in error_analysis: - suggestions.extend([ - "Check file permissions on the target path", - "Ensure the Docker daemon has access to the mounted volumes", - "Try running with elevated privileges or adjust volume ownership" - ]) - - if 'out_of_memory' in error_analysis: - suggestions.extend([ - "Increase memory limits for the workflow", - "Check if the target files are too large for available memory", - "Consider using streaming processing for large datasets" - ]) - - if 'image_pull_failed' in error_analysis: - suggestions.extend([ - "Check network connectivity to Docker registry", - "Verify image name and tag are correct", - "Ensure Docker registry credentials are configured" - ]) - - if 'volume_mount_failed' in error_analysis: - suggestions.extend([ - "Verify the target path exists and is accessible", - "Check volume mount syntax and permissions", - "Ensure the path is not already in use by another process" - ]) - - if 'network_error' in error_analysis: - suggestions.extend([ - "Check network connectivity", - "Verify backend services are running (docker-compose up -d)", - "Check firewall settings and port availability" - ]) - - if 'prefect_error' in error_analysis: - suggestions.extend([ - "Check Prefect server connectivity", - "Verify workflow deployment is successful", - "Review workflow-specific parameters and requirements" - ]) - - if not suggestions: - suggestions.append("Review the container logs above for specific error details") - - return suggestions - - -# Global instance for easy access -docker_integration = DockerLogIntegration() \ No newline at end of file diff --git a/sdk/src/fuzzforge_sdk/exceptions.py b/sdk/src/fuzzforge_sdk/exceptions.py index 03b34a5..c587658 100644 --- a/sdk/src/fuzzforge_sdk/exceptions.py +++ b/sdk/src/fuzzforge_sdk/exceptions.py @@ -1,8 +1,9 @@ """ -Enhanced exceptions for FuzzForge SDK with rich context and Docker integration. +Enhanced exceptions for FuzzForge SDK with rich context. -Provides comprehensive error information including container logs, diagnostics, -and actionable suggestions for troubleshooting. +Provides comprehensive error information and actionable suggestions for troubleshooting. +Note: Container diagnostics are not available in Temporal architecture as workflows +run in long-lived worker containers rather than ephemeral per-workflow containers. """ # Copyright (c) 2025 FuzzingLabs # @@ -18,11 +19,9 @@ and actionable suggestions for troubleshooting. import json import re -from typing import Optional, Dict, Any, List, Union +from typing import Optional, Dict, Any, List from dataclasses import dataclass, asdict -from .docker_logs import docker_integration, ContainerDiagnostics - @dataclass class ErrorContext: @@ -31,7 +30,6 @@ class ErrorContext: request_method: Optional[str] = None request_data: Optional[Dict[str, Any]] = None response_data: Optional[Dict[str, Any]] = None - container_diagnostics: Optional[ContainerDiagnostics] = None suggested_fixes: List[str] = None error_patterns: Dict[str, List[str]] = None related_run_id: Optional[str] = None @@ -62,49 +60,10 @@ class FuzzForgeError(Exception): self.context = context or ErrorContext() self.original_exception = original_exception - # Auto-populate container diagnostics if we have a run ID - if self.context.related_run_id and not self.context.container_diagnostics: - self._fetch_container_diagnostics() - - def _fetch_container_diagnostics(self): - """Fetch container diagnostics for the related run.""" - if not self.context.related_run_id: - return - - try: - # Try to find containers by Prefect run ID label - label_filter = f"prefect.flow-run-id={self.context.related_run_id}" - container_names = docker_integration.get_container_names_by_label(label_filter) - - if container_names: - # Use the most recent container - container_name = container_names[0] - diagnostics = docker_integration.get_container_diagnostics(container_name) - - # Analyze error patterns in logs - if diagnostics.logs: - error_analysis = docker_integration.analyze_error_patterns(diagnostics.logs) - suggestions = docker_integration.suggest_fixes(error_analysis) - - self.context.container_diagnostics = diagnostics - self.context.error_patterns = error_analysis - self.context.suggested_fixes.extend(suggestions) - - except Exception: - # Don't fail the main error because of diagnostics issues - pass - def get_summary(self) -> str: """Get a summary of the error with key details.""" parts = [self.message] - if self.context.container_diagnostics: - diag = self.context.container_diagnostics - if diag.status != 'running': - parts.append(f"Container status: {diag.status}") - if diag.exit_code is not None: - parts.append(f"Exit code: {diag.exit_code}") - if self.context.error_patterns: detected = list(self.context.error_patterns.keys()) parts.append(f"Detected issues: {', '.join(detected)}") @@ -153,18 +112,11 @@ class FuzzForgeHTTPError(FuzzForgeError): self.response_text = response_text def get_summary(self) -> str: - base = f"HTTP {self.status_code}: {self.message}" - - if self.context.container_diagnostics: - diag = self.context.container_diagnostics - if diag.exit_code is not None and diag.exit_code != 0: - base += f" (Container exit code: {diag.exit_code})" - - return base + return f"HTTP {self.status_code}: {self.message}" class DeploymentError(FuzzForgeHTTPError): - """Enhanced deployment errors with container diagnostics.""" + """Enhanced deployment errors.""" def __init__( self, @@ -181,23 +133,9 @@ class DeploymentError(FuzzForgeHTTPError): context.workflow_name = workflow_name - # If we have a container name, get its diagnostics immediately - if container_name: - try: - diagnostics = docker_integration.get_container_diagnostics(container_name) - context.container_diagnostics = diagnostics - - # Analyze logs for error patterns - if diagnostics.logs: - error_analysis = docker_integration.analyze_error_patterns(diagnostics.logs) - suggestions = docker_integration.suggest_fixes(error_analysis) - - context.error_patterns = error_analysis - context.suggested_fixes.extend(suggestions) - - except Exception: - # Don't fail on diagnostics - pass + # Note: Container diagnostics are not fetched in Temporal architecture. + # Workflows run in long-lived worker containers, not per-workflow containers. + # The container_name parameter is kept for backward compatibility but not used. full_message = f"Deployment failed for workflow '{workflow_name}': {message}" super().__init__(full_message, status_code, response_text, context) @@ -292,22 +230,9 @@ class ContainerError(FuzzForgeError): if context is None: context = ErrorContext() - # Immediately fetch container diagnostics - try: - diagnostics = docker_integration.get_container_diagnostics(container_name) - context.container_diagnostics = diagnostics - - # Analyze logs for patterns - if diagnostics.logs: - error_analysis = docker_integration.analyze_error_patterns(diagnostics.logs) - suggestions = docker_integration.suggest_fixes(error_analysis) - - context.error_patterns = error_analysis - context.suggested_fixes.extend(suggestions) - - except Exception: - # Don't fail on diagnostics - pass + # Note: Container diagnostics are not fetched in Temporal architecture. + # Workflows run in long-lived worker containers, not per-workflow containers. + # The container_name parameter is kept for backward compatibility but not used. full_message = f"Container error ({container_name}): {message}" if exit_code is not None: diff --git a/sdk/src/fuzzforge_sdk/models.py b/sdk/src/fuzzforge_sdk/models.py index 1ad5a25..e92f75e 100644 --- a/sdk/src/fuzzforge_sdk/models.py +++ b/sdk/src/fuzzforge_sdk/models.py @@ -16,49 +16,18 @@ and serialization for all API requests and responses. # Additional attribution and requirements are provided in the NOTICE file. -from pydantic import BaseModel, Field, validator -from typing import Dict, Any, Optional, Literal, List, Union +from pydantic import BaseModel, Field +from typing import Dict, Any, Optional, List, Union from datetime import datetime -from pathlib import Path - - -class ResourceLimits(BaseModel): - """Resource limits for workflow execution""" - cpu_limit: Optional[str] = Field(None, description="CPU limit (e.g., '2' for 2 cores, '500m' for 0.5 cores)") - memory_limit: Optional[str] = Field(None, description="Memory limit (e.g., '1Gi', '512Mi')") - cpu_request: Optional[str] = Field(None, description="CPU request (guaranteed)") - memory_request: Optional[str] = Field(None, description="Memory request (guaranteed)") - - -class VolumeMount(BaseModel): - """Volume mount specification""" - host_path: str = Field(..., description="Host path to mount") - container_path: str = Field(..., description="Container path for mount") - mode: Literal["ro", "rw"] = Field(default="ro", description="Mount mode") - - @validator("host_path") - def validate_host_path(cls, v): - """Validate that the host path is absolute""" - path = Path(v) - if not path.is_absolute(): - raise ValueError(f"Host path must be absolute: {v}") - return str(path) - - @validator("container_path") - def validate_container_path(cls, v): - """Validate that the container path is absolute""" - if not v.startswith('/'): - raise ValueError(f"Container path must be absolute: {v}") - return v class WorkflowSubmission(BaseModel): - """Submit a workflow with configurable settings""" - target_path: str = Field(..., description="Absolute path to analyze") - volume_mode: Literal["ro", "rw"] = Field( - default="ro", - description="Volume mount mode: read-only (ro) or read-write (rw)" - ) + """ + Submit a workflow with configurable settings. + + Note: This model is deprecated in favor of direct file upload via + submit_workflow_with_upload() which handles file uploads automatically. + """ parameters: Dict[str, Any] = Field( default_factory=dict, description="Workflow-specific parameters" @@ -69,22 +38,6 @@ class WorkflowSubmission(BaseModel): ge=1, le=604800 # Max 7 days ) - resource_limits: Optional[ResourceLimits] = Field( - None, - description="Resource limits for workflow container" - ) - additional_volumes: List[VolumeMount] = Field( - default_factory=list, - description="Additional volume mounts" - ) - - @validator("target_path") - def validate_path(cls, v): - """Validate that the target path is absolute""" - path = Path(v) - if not path.is_absolute(): - raise ValueError(f"Path must be absolute: {v}") - return str(path) class WorkflowListItem(BaseModel): @@ -112,10 +65,6 @@ class WorkflowMetadata(BaseModel): default_factory=list, description="Required module names" ) - supported_volume_modes: List[Literal["ro", "rw"]] = Field( - default=["ro", "rw"], - description="Supported volume mount modes" - ) has_custom_docker: bool = Field( default=False, description="Whether workflow has custom Dockerfile" @@ -124,9 +73,10 @@ class WorkflowMetadata(BaseModel): class WorkflowParametersResponse(BaseModel): """Response for workflow parameters endpoint""" + workflow: str = Field(..., description="Workflow name") parameters: Dict[str, Any] = Field(..., description="Parameters schema") - defaults: Dict[str, Any] = Field(default_factory=dict, description="Default values") - required: List[str] = Field(default_factory=list, description="Required parameter names") + default_parameters: Dict[str, Any] = Field(default_factory=dict, description="Default parameter values") + required_parameters: List[str] = Field(default_factory=list, description="Required parameter names") class RunSubmissionResponse(BaseModel): diff --git a/sdk/src/fuzzforge_sdk/testing.py b/sdk/src/fuzzforge_sdk/testing.py index 67457ea..9f9297b 100644 --- a/sdk/src/fuzzforge_sdk/testing.py +++ b/sdk/src/fuzzforge_sdk/testing.py @@ -18,15 +18,14 @@ workflow functionality, performance, and expected results. import time from pathlib import Path -from typing import Dict, Any, List, Optional, Union +from typing import Dict, Any, List, Optional from dataclasses import dataclass from datetime import datetime import logging from .client import FuzzForgeClient -from .models import WorkflowSubmission from .utils import validate_absolute_path, create_workflow_submission -from .exceptions import FuzzForgeError, ValidationError +from .exceptions import ValidationError logger = logging.getLogger(__name__) diff --git a/sdk/src/fuzzforge_sdk/utils.py b/sdk/src/fuzzforge_sdk/utils.py index 10614cb..7ff19b2 100644 --- a/sdk/src/fuzzforge_sdk/utils.py +++ b/sdk/src/fuzzforge_sdk/utils.py @@ -20,9 +20,8 @@ import os import json from pathlib import Path from typing import Dict, Any, List, Optional, Union -from datetime import datetime -from .models import VolumeMount, ResourceLimits, WorkflowSubmission +from .models import WorkflowSubmission from .exceptions import ValidationError @@ -50,112 +49,19 @@ def validate_absolute_path(path: Union[str, Path]) -> Path: return path_obj -def create_volume_mount( - host_path: Union[str, Path], - container_path: str, - mode: str = "ro" -) -> VolumeMount: - """ - Create a volume mount with path validation. - - Args: - host_path: Host path to mount (must exist) - container_path: Container path for the mount - mode: Mount mode ("ro" or "rw") - - Returns: - VolumeMount object - - Raises: - ValidationError: If paths are invalid - """ - # Validate host path exists and is absolute - validated_host_path = validate_absolute_path(host_path) - - # Validate container path is absolute - if not container_path.startswith('/'): - raise ValidationError(f"Container path must be absolute: {container_path}") - - # Validate mode - if mode not in ["ro", "rw"]: - raise ValidationError(f"Mode must be 'ro' or 'rw': {mode}") - - return VolumeMount( - host_path=str(validated_host_path), - container_path=container_path, - mode=mode # type: ignore - ) - - -def create_resource_limits( - cpu_limit: Optional[str] = None, - memory_limit: Optional[str] = None, - cpu_request: Optional[str] = None, - memory_request: Optional[str] = None -) -> ResourceLimits: - """ - Create resource limits with validation. - - Args: - cpu_limit: CPU limit (e.g., "2", "500m") - memory_limit: Memory limit (e.g., "1Gi", "512Mi") - cpu_request: CPU request (guaranteed) - memory_request: Memory request (guaranteed) - - Returns: - ResourceLimits object - - Raises: - ValidationError: If resource specifications are invalid - """ - # Basic validation for CPU limits - if cpu_limit is not None: - if not (cpu_limit.endswith('m') or cpu_limit.isdigit()): - raise ValidationError(f"Invalid CPU limit format: {cpu_limit}") - - if cpu_request is not None: - if not (cpu_request.endswith('m') or cpu_request.isdigit()): - raise ValidationError(f"Invalid CPU request format: {cpu_request}") - - # Basic validation for memory limits - memory_suffixes = ['Ki', 'Mi', 'Gi', 'Ti', 'K', 'M', 'G', 'T'] - - if memory_limit is not None: - if not any(memory_limit.endswith(suffix) for suffix in memory_suffixes): - if not memory_limit.isdigit(): - raise ValidationError(f"Invalid memory limit format: {memory_limit}") - - if memory_request is not None: - if not any(memory_request.endswith(suffix) for suffix in memory_suffixes): - if not memory_request.isdigit(): - raise ValidationError(f"Invalid memory request format: {memory_request}") - - return ResourceLimits( - cpu_limit=cpu_limit, - memory_limit=memory_limit, - cpu_request=cpu_request, - memory_request=memory_request - ) - - def create_workflow_submission( - target_path: Union[str, Path], - volume_mode: str = "ro", parameters: Optional[Dict[str, Any]] = None, - timeout: Optional[int] = None, - resource_limits: Optional[ResourceLimits] = None, - additional_volumes: Optional[List[VolumeMount]] = None + timeout: Optional[int] = None ) -> WorkflowSubmission: """ - Create a workflow submission with path validation. + Create a workflow submission. + + Note: This function is deprecated. Use client.submit_workflow_with_upload() instead + which handles file uploads automatically. Args: - target_path: Path to analyze (must exist) - volume_mode: Mount mode for target path parameters: Workflow-specific parameters timeout: Execution timeout in seconds - resource_limits: Resource limits for the container - additional_volumes: Additional volume mounts Returns: WorkflowSubmission object @@ -163,25 +69,14 @@ def create_workflow_submission( Raises: ValidationError: If parameters are invalid """ - # Validate target path - validated_target_path = validate_absolute_path(target_path) - - # Validate volume mode - if volume_mode not in ["ro", "rw"]: - raise ValidationError(f"Volume mode must be 'ro' or 'rw': {volume_mode}") - # Validate timeout if timeout is not None: if timeout < 1 or timeout > 604800: # Max 7 days raise ValidationError(f"Timeout must be between 1 and 604800 seconds: {timeout}") return WorkflowSubmission( - target_path=str(validated_target_path), - volume_mode=volume_mode, # type: ignore parameters=parameters or {}, - timeout=timeout, - resource_limits=resource_limits, - additional_volumes=additional_volumes or [] + timeout=timeout ) diff --git a/sdk/uv.lock b/sdk/uv.lock index 90ce961..f332848 100644 --- a/sdk/uv.lock +++ b/sdk/uv.lock @@ -85,7 +85,7 @@ wheels = [ [[package]] name = "fuzzforge-sdk" -version = "0.1.0" +version = "0.6.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/setup.py b/setup.py index d905ade..2438594 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ import subprocess import platform import time from pathlib import Path -from typing import List, Tuple, Optional +from typing import List, Tuple class Colors: diff --git a/test_projects/README.md b/test_projects/README.md index 616978c..70cfbcd 100644 --- a/test_projects/README.md +++ b/test_projects/README.md @@ -1,6 +1,6 @@ # FuzzForge Vulnerable Test Project -This directory contains a comprehensive vulnerable test application designed to validate FuzzForge's security workflows. The project contains multiple categories of security vulnerabilities to test both the `security_assessment` and `secret_detection_scan` workflows. +This directory contains a comprehensive vulnerable test application designed to validate FuzzForge's security workflows. The project contains multiple categories of security vulnerabilities to test `security_assessment`, `gitleaks_detection`, `trufflehog_detection`, and `llm_secret_detection` workflows. ## Test Project Overview @@ -9,7 +9,9 @@ This directory contains a comprehensive vulnerable test application designed to **Supported Workflows**: - `security_assessment` - General security scanning and analysis -- `secret_detection_scan` - Detection of secrets, credentials, and sensitive data +- `gitleaks_detection` - Pattern-based secret detection +- `trufflehog_detection` - Entropy-based secret detection with verification +- `llm_secret_detection` - AI-powered semantic secret detection **Vulnerabilities Included**: - SQL injection vulnerabilities @@ -38,23 +40,28 @@ This directory contains a comprehensive vulnerable test application designed to ### Testing with FuzzForge Workflows -The vulnerable application can be tested with both essential workflows: +The vulnerable application can be tested with multiple security workflows: ```bash # Test security assessment workflow curl -X POST http://localhost:8000/workflows/security_assessment/submit \ -H "Content-Type: application/json" \ -d '{ - "target_path": "/path/to/test_projects/vulnerable_app", - "volume_mode": "ro" + "target_path": "/path/to/test_projects/vulnerable_app" }' -# Test secret detection workflow -curl -X POST http://localhost:8000/workflows/secret_detection_scan/submit \ +# Test Gitleaks secret detection workflow +curl -X POST http://localhost:8000/workflows/gitleaks_detection/submit \ -H "Content-Type: application/json" \ -d '{ - "target_path": "/path/to/test_projects/vulnerable_app", - "volume_mode": "ro" + "target_path": "/path/to/test_projects/vulnerable_app" + }' + +# Test TruffleHog secret detection workflow +curl -X POST http://localhost:8000/workflows/trufflehog_detection/submit \ + -H "Content-Type: application/json" \ + -d '{ + "target_path": "/path/to/test_projects/vulnerable_app" }' ``` @@ -70,7 +77,9 @@ Each workflow should produce SARIF-formatted results with: A successful test should detect: - **Security Assessment**: At least 20 various security vulnerabilities -- **Secret Detection**: At least 10 different types of secrets and credentials +- **Gitleaks Detection**: At least 10 different types of secrets +- **TruffleHog Detection**: At least 5 high-entropy secrets +- **LLM Secret Detection**: At least 15 secrets with semantic understanding --- diff --git a/test_projects/python_fuzz_waterfall/.gitignore b/test_projects/python_fuzz_waterfall/.gitignore new file mode 100644 index 0000000..ad398c8 --- /dev/null +++ b/test_projects/python_fuzz_waterfall/.gitignore @@ -0,0 +1,18 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ + +# FuzzForge +.fuzzforge/ + +# Atheris fuzzing artifacts +corpus/ +crashes/ +*.profraw +*.profdata diff --git a/test_projects/python_fuzz_waterfall/README.md b/test_projects/python_fuzz_waterfall/README.md new file mode 100644 index 0000000..2987ed9 --- /dev/null +++ b/test_projects/python_fuzz_waterfall/README.md @@ -0,0 +1,137 @@ +# Python Fuzzing Test - Waterfall Vulnerability + +This project demonstrates a **stateful vulnerability** that Atheris can discover through fuzzing. + +## Vulnerability Description + +The `check_secret()` function in `main.py` validates input character by character against the secret string "FUZZINGLABS". This creates a **waterfall vulnerability** where: + +1. State leaks through the global `progress` variable +2. Each correct character advances the progress counter +3. When all 11 characters are provided in order, the function crashes with `SystemError` + +This pattern is analogous to: +- Timing attacks on password checkers +- Protocol state machines with sequential validation +- Multi-step authentication flows + +## Files + +- `main.py` - Main application with vulnerable `check_secret()` function +- `fuzz_target.py` - Atheris fuzzing harness (contains `TestOneInput()`) +- `README.md` - This file + +## How to Fuzz + +### Using FuzzForge CLI + +```bash +# Initialize FuzzForge in this directory +cd test_projects/python_fuzz_waterfall/ +ff init + +# Run fuzzing workflow (uploads code to MinIO) +ff workflow run atheris_fuzzing . + +# The workflow will: +# 1. Upload this directory to MinIO +# 2. Worker downloads and extracts the code +# 3. Worker discovers fuzz_target.py (has TestOneInput) +# 4. Worker runs Atheris fuzzing +# 5. Reports real-time stats every 5 seconds +# 6. Finds crash when "FUZZINGLABS" is discovered +``` + +### Using FuzzForge SDK + +```python +from fuzzforge_sdk import FuzzForgeClient +from pathlib import Path + +client = FuzzForgeClient(base_url="http://localhost:8000") + +# Upload and run fuzzing +response = client.submit_workflow_with_upload( + workflow_name="atheris_fuzzing", + target_path=Path("./"), + parameters={ + "max_iterations": 100000, + "timeout_seconds": 300 + } +) + +print(f"Workflow started: {response.run_id}") + +# Wait for completion +final_status = client.wait_for_completion(response.run_id) +findings = client.get_run_findings(response.run_id) + +for finding in findings: + print(f"Crash: {finding.title}") + print(f"Input: {finding.metadata.get('crash_input_hex')}") +``` + +### Standalone (Without FuzzForge) + +```bash +# Install Atheris +pip install atheris + +# Run fuzzing directly +python fuzz_target.py +``` + +## Expected Behavior + +When fuzzing: + +1. **Initial phase**: Random exploration, progress = 0 +2. **Discovery phase**: Atheris finds 'F' (first char), progress = 1 +3. **Incremental progress**: Finds 'U', then 'Z', etc. +4. **Crash**: When full "FUZZINGLABS" discovered, crashes with: + ``` + SystemError: SECRET COMPROMISED: FUZZINGLABS + ``` + +## Monitoring + +Watch real-time fuzzing stats: + +```bash +docker logs fuzzforge-worker-python -f | grep LIVE_STATS +``` + +Output example: +``` +INFO - LIVE_STATS - executions=1523 execs_per_sec=1523.0 crashes=0 +INFO - LIVE_STATS - executions=7842 execs_per_sec=2104.2 crashes=0 +INFO - LIVE_STATS - executions=15234 execs_per_sec=2167.0 crashes=1 ā† Crash found! +``` + +## Vulnerability Details + +**CVE**: N/A (demonstration vulnerability) +**CWE**: CWE-208 (Observable Timing Discrepancy) +**Severity**: Critical (in real systems) + +**Fix**: Remove state-based checking or implement constant-time comparison: + +```python +def check_secret_safe(input_data: bytes) -> bool: + """Constant-time comparison""" + import hmac + return hmac.compare_digest(input_data, SECRET.encode()) +``` + +## Adjusting Difficulty + +If fuzzing finds the crash too quickly, extend the secret: + +```python +# In main.py, change: +SECRET = "FUZZINGLABSSECURITYTESTING" # 26 characters instead of 11 +``` + +## License + +MIT License - This is a demonstration project for educational purposes. diff --git a/test_projects/python_fuzz_waterfall/fuzz_target.py b/test_projects/python_fuzz_waterfall/fuzz_target.py new file mode 100644 index 0000000..a07faba --- /dev/null +++ b/test_projects/python_fuzz_waterfall/fuzz_target.py @@ -0,0 +1,62 @@ +""" +Atheris fuzzing target for the waterfall vulnerability. + +This file is automatically discovered by FuzzForge's AtherisFuzzer module. +The fuzzer looks for files named: fuzz_*.py, *_fuzz.py, or fuzz_target.py +""" + +import sys +import atheris + +# Enable coverage instrumentation for imported modules +# This is critical for discovering the waterfall vulnerability! +with atheris.instrument_imports(): + from main import check_secret + + +def TestOneInput(data): + """ + Atheris fuzzing entry point. + + This function is called by Atheris for each fuzzing iteration. + The fuzzer will try to find inputs that cause crashes. + + Args: + data: Bytes to test (generated by Atheris) + + The waterfall vulnerability: + - check_secret() validates input character-by-character + - Each correct character creates a distinct code path + - Coverage-guided fuzzing progressively discovers the secret "FUZZINGLABS" + - When the complete secret is found, it crashes with SystemError + + With atheris.instrument_imports(), the main module is instrumented + for coverage, allowing Atheris to detect when inputs reach new + code paths (each correct character). + """ + # Call the vulnerable function + # It will raise SystemError when the secret is fully discovered + check_secret(bytes(data)) + + +if __name__ == "__main__": + """ + Standalone fuzzing mode. + + Run directly: python fuzz_target.py + """ + print("=" * 60) + print("Atheris Fuzzing - Waterfall Vulnerability") + print("=" * 60) + print("Fuzzing will try to discover the secret string...") + print("Watch for progress indicators: [DEBUG] Progress: X/11") + print() + print("Press Ctrl+C to stop fuzzing") + print("=" * 60) + print() + + # Setup Atheris with command-line args + atheris.Setup(sys.argv, TestOneInput) + + # Start fuzzing + atheris.Fuzz() diff --git a/test_projects/python_fuzz_waterfall/main.py b/test_projects/python_fuzz_waterfall/main.py new file mode 100644 index 0000000..4042a2a --- /dev/null +++ b/test_projects/python_fuzz_waterfall/main.py @@ -0,0 +1,117 @@ +""" +Example application with a waterfall vulnerability. + +This simulates a password checking system that validates character-by-character. +Each correct character creates a distinct code path, allowing coverage-guided +fuzzing to progressively discover the secret. +""" + +SECRET = b"FUZZINGLABS" # Full secret to discover + + +def check_secret(input_data: bytes) -> int: + """ + Vulnerable function: checks secret character by character. + + This is a classic waterfall/sequential comparison vulnerability. + Each correct character comparison creates a unique code path that + coverage-guided fuzzing can detect and use to guide input generation. + + Real-world analogy: + - Timing attacks on password checkers + - Protocol state machines with sequential validation + - JWT signature verification vulnerabilities + + Args: + input_data: Input bytes to check + + Returns: + Number of matching characters (for instrumentation purposes) + + Raises: + SystemError: When complete secret is discovered + """ + if not input_data: + return 0 + + # Check each character sequentially + # Each comparison creates a distinct code path for coverage guidance + matches = 0 + for i in range(min(len(input_data), len(SECRET))): + if input_data[i] != SECRET[i]: + # Wrong character - stop checking + return matches + + matches += 1 + + # Add explicit comparisons to help coverage-guided fuzzing + # Each comparison creates a distinct code path for Atheris to detect + if matches >= 1 and input_data[0] == ord('F'): + pass # F + if matches >= 2 and input_data[1] == ord('U'): + pass # FU + if matches >= 3 and input_data[2] == ord('Z'): + pass # FUZ + if matches >= 4 and input_data[3] == ord('Z'): + pass # FUZZ + if matches >= 5 and input_data[4] == ord('I'): + pass # FUZZI + if matches >= 6 and input_data[5] == ord('N'): + pass # FUZZIN + if matches >= 7 and input_data[6] == ord('G'): + pass # FUZZING + if matches >= 8 and input_data[7] == ord('L'): + pass # FUZZINGL + if matches >= 9 and input_data[8] == ord('A'): + pass # FUZZINGLA + if matches >= 10 and input_data[9] == ord('B'): + pass # FUZZINGLAB + if matches >= 11 and input_data[10] == ord('S'): + pass # FUZZINGLABS + + # VULNERABILITY: Crashes when complete secret found + if matches == len(SECRET) and len(input_data) >= len(SECRET): + raise SystemError(f"SECRET COMPROMISED! Found: {input_data[:len(SECRET)]}") + + return matches + + +def reset_state(): + """Reset the global state (kept for compatibility, but not used)""" + pass + + +if __name__ == "__main__": + """Example usage showing the vulnerability""" + print("=" * 60) + print("Waterfall Vulnerability Demonstration") + print("=" * 60) + print(f"Secret: {SECRET}") + print(f"Secret length: {len(SECRET)} characters") + print() + + # Test inputs showing progressive discovery + test_inputs = [ + b"F", # First char correct + b"FU", # First two chars correct + b"FUZ", # First three chars correct + b"WRONG", # Wrong - no matches + b"FUZZINGLABS", # Complete secret - triggers crash! + ] + + for test in test_inputs: + print(f"Testing input: {test.decode(errors='ignore')!r}") + + try: + matches = check_secret(test) + print(f" Result: {matches} characters matched out of {len(SECRET)}") + except SystemError as e: + print(f" šŸ’„ CRASH: {e}") + + print() + + print("=" * 60) + print("To fuzz this vulnerability with FuzzForge:") + print(" ff init") + print(" ff workflow run atheris_fuzzing .") + print("=" * 60) diff --git a/test_projects/rust_fuzz_test/Cargo.toml b/test_projects/rust_fuzz_test/Cargo.toml new file mode 100644 index 0000000..3d3fe2b --- /dev/null +++ b/test_projects/rust_fuzz_test/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "rust_fuzz_test" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/test_projects/rust_fuzz_test/README.md b/test_projects/rust_fuzz_test/README.md new file mode 100644 index 0000000..67c36fb --- /dev/null +++ b/test_projects/rust_fuzz_test/README.md @@ -0,0 +1,22 @@ +# rust_fuzz_test + +FuzzForge security testing project. + +## Quick Start + +```bash +# List available workflows +fuzzforge workflows + +# Submit a workflow for analysis +fuzzforge workflow /path/to/target + +# View findings +fuzzforge finding +``` + +## Project Structure + +- `.fuzzforge/` - Project data and configuration +- `.fuzzforge/config.yaml` - Project configuration +- `.fuzzforge/findings.db` - Local database for runs and findings diff --git a/test_projects/rust_fuzz_test/fuzz/.gitignore b/test_projects/rust_fuzz_test/fuzz/.gitignore new file mode 100644 index 0000000..1a45eee --- /dev/null +++ b/test_projects/rust_fuzz_test/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/test_projects/rust_fuzz_test/fuzz/Cargo.toml b/test_projects/rust_fuzz_test/fuzz/Cargo.toml new file mode 100644 index 0000000..b28a2c1 --- /dev/null +++ b/test_projects/rust_fuzz_test/fuzz/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "rust_fuzz_test-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.rust_fuzz_test] +path = ".." + +[[bin]] +name = "fuzz_target_1" +path = "fuzz_targets/fuzz_target_1.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "fuzz_divide" +path = "fuzz_targets/fuzz_divide.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "fuzz_waterfall" +path = "fuzz_targets/fuzz_waterfall.rs" +test = false +doc = false +bench = false diff --git a/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_divide.rs b/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_divide.rs new file mode 100644 index 0000000..78ba872 --- /dev/null +++ b/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_divide.rs @@ -0,0 +1,9 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rust_fuzz_test::divide_numbers; + +fuzz_target!(|data: &[u8]| { + // Fuzz the divide_numbers function which has division by zero + let _ = divide_numbers(data); +}); diff --git a/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_target_1.rs b/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_target_1.rs new file mode 100644 index 0000000..9f893e1 --- /dev/null +++ b/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_target_1.rs @@ -0,0 +1,9 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rust_fuzz_test::process_buffer; + +fuzz_target!(|data: &[u8]| { + // Fuzz the process_buffer function which has bounds checking issues + let _ = process_buffer(data); +}); diff --git a/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_waterfall.rs b/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_waterfall.rs new file mode 100644 index 0000000..fec90be --- /dev/null +++ b/test_projects/rust_fuzz_test/fuzz/fuzz_targets/fuzz_waterfall.rs @@ -0,0 +1,9 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use rust_fuzz_test::check_secret_waterfall; + +fuzz_target!(|data: &[u8]| { + // Fuzz the waterfall vulnerability - sequential secret checking + let _ = check_secret_waterfall(data); +}); diff --git a/test_projects/rust_fuzz_test/src/lib.rs b/test_projects/rust_fuzz_test/src/lib.rs new file mode 100644 index 0000000..3f271d9 --- /dev/null +++ b/test_projects/rust_fuzz_test/src/lib.rs @@ -0,0 +1,138 @@ +/// Parse a simple integer from bytes +/// This function has a potential panic if the input is invalid +pub fn parse_number(data: &[u8]) -> i32 { + let s = std::str::from_utf8(data).expect("Invalid UTF-8"); + s.parse::().expect("Invalid number") +} + +/// Process a buffer with bounds checking issue +pub fn process_buffer(data: &[u8]) -> Vec { + if data.len() < 4 { + return Vec::new(); + } + + // Only crash when specific conditions are met (makes it harder to find) + if data[0] == b'F' && data[1] == b'U' && data[2] == b'Z' && data[3] == b'Z' { + // Potential panic: accessing index without proper bounds check + let size = data[4] as usize; // Will panic if data.len() == 4 + let mut result = Vec::new(); + + // This could panic if size is larger than data.len() + for i in 4..4+size { + result.push(data[i]); // Will panic if i >= data.len() + } + + return result; + } + + Vec::new() +} + +/// Divide two numbers parsed from input +pub fn divide_numbers(data: &[u8]) -> Option { + if data.len() < 2 { + return None; + } + + let a = data[0] as i32; + let b = data[1] as i32; + + // Potential division by zero + Some(a / b) +} + +/// Waterfall vulnerability: checks secret character by character +/// This is a classic sequential comparison vulnerability that creates +/// distinct code paths for coverage-guided fuzzing to discover. +pub fn check_secret_waterfall(data: &[u8]) -> usize { + const SECRET: &[u8] = b"FUZZINGLABS"; + + if data.is_empty() { + return 0; + } + + let mut matches = 0; + + // Check each character sequentially + // Each comparison creates a distinct code path for coverage guidance + for i in 0..std::cmp::min(data.len(), SECRET.len()) { + if data[i] != SECRET[i] { + // Wrong character - stop checking + return matches; + } + + matches += 1; + + // Add explicit comparisons to help coverage-guided fuzzing + // Each comparison creates a distinct code path for the fuzzer to detect + if matches >= 1 && data[0] == b'F' { + // F + } + if matches >= 2 && data[1] == b'U' { + // FU + } + if matches >= 3 && data[2] == b'Z' { + // FUZ + } + if matches >= 4 && data[3] == b'Z' { + // FUZZ + } + if matches >= 5 && data[4] == b'I' { + // FUZZI + } + if matches >= 6 && data[5] == b'N' { + // FUZZIN + } + if matches >= 7 && data[6] == b'G' { + // FUZZING + } + if matches >= 8 && data[7] == b'L' { + // FUZZINGL + } + if matches >= 9 && data[8] == b'A' { + // FUZZINGLA + } + if matches >= 10 && data[9] == b'B' { + // FUZZINGLAB + } + if matches >= 11 && data[10] == b'S' { + // FUZZINGLABS + } + } + + // VULNERABILITY: Panics when complete secret found + if matches == SECRET.len() && data.len() >= SECRET.len() { + panic!("SECRET COMPROMISED! Found: {:?}", &data[..SECRET.len()]); + } + + matches +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_number() { + assert_eq!(parse_number(b"123"), 123); + } + + #[test] + fn test_process_buffer() { + let data = vec![3, 1, 2, 3, 4]; + assert_eq!(process_buffer(&data), vec![3, 1, 2]); + } + + #[test] + fn test_waterfall_partial_match() { + assert_eq!(check_secret_waterfall(b"F"), 1); + assert_eq!(check_secret_waterfall(b"FU"), 2); + assert_eq!(check_secret_waterfall(b"FUZZ"), 4); + } + + #[test] + #[should_panic(expected = "SECRET COMPROMISED")] + fn test_waterfall_full_match() { + check_secret_waterfall(b"FUZZINGLABS"); + } +} diff --git a/test_projects/secret_detection_benchmark/.env b/test_projects/secret_detection_benchmark/.env new file mode 100644 index 0000000..ac08a7c --- /dev/null +++ b/test_projects/secret_detection_benchmark/.env @@ -0,0 +1,7 @@ +# Environment configuration +# EASY SECRET #1: Plain AWS access key +AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +DATABASE_HOST=localhost +DATABASE_PORT=5432 diff --git a/test_projects/secret_detection_benchmark/.fuzzforge/findings.db b/test_projects/secret_detection_benchmark/.fuzzforge/findings.db new file mode 100644 index 0000000..8bc12ca Binary files /dev/null and b/test_projects/secret_detection_benchmark/.fuzzforge/findings.db differ diff --git a/test_projects/secret_detection_benchmark/README.md b/test_projects/secret_detection_benchmark/README.md new file mode 100644 index 0000000..2101179 --- /dev/null +++ b/test_projects/secret_detection_benchmark/README.md @@ -0,0 +1,99 @@ +# Secret Detection Benchmark Dataset + +Ground truth dataset with **exactly 32 known secrets** for testing secret detection tools. + +## Contents + +- **12 Easy Secrets**: Standard patterns (AWS keys, GitHub PATs, Stripe keys, etc.) +- **10 Medium Secrets**: Slightly obfuscated (Base64, hex, concatenated, in comments) +- **10 Hard Secrets**: Well hidden (ROT13, binary, XOR, reversed, template strings) + +## Files + +``` +ā”œā”€ā”€ .env # 2 secrets +ā”œā”€ā”€ config/ +ā”‚ ā”œā”€ā”€ settings.py # 3 secrets +ā”‚ ā”œā”€ā”€ database.yaml # 1 secret +ā”‚ ā”œā”€ā”€ app.properties # 1 secret +ā”‚ ā”œā”€ā”€ oauth.json # 1 secret +ā”‚ ā”œā”€ā”€ keys.yaml # 2 secrets +ā”‚ ā””ā”€ā”€ legacy.ini # 2 secrets +ā”œā”€ā”€ src/ +ā”‚ ā”œā”€ā”€ app.py # 1 secret +ā”‚ ā”œā”€ā”€ Main.java # 1 secret +ā”‚ ā”œā”€ā”€ config.py # 3 secrets (medium difficulty) +ā”‚ ā”œā”€ā”€ obfuscated.py # 4 secrets (hard difficulty) +ā”‚ ā”œā”€ā”€ advanced.js # 4 secrets (hard difficulty) +ā”‚ ā”œā”€ā”€ Crypto.go # 2 secrets (hard difficulty) +ā”‚ ā””ā”€ā”€ database.sql # 1 secret +ā”œā”€ā”€ scripts/ +ā”‚ ā”œā”€ā”€ webhook.js # 1 secret +ā”‚ ā””ā”€ā”€ deploy.sh # 2 secrets +ā””ā”€ā”€ id_rsa # 1 secret + +Total: 17 files with 32 secrets +``` + +## Secret Difficulty Breakdown + +### Easy (12 secrets) +Should be detected by any decent secret scanner: +- Plain AWS access keys +- GitHub Personal Access Tokens +- Stripe API keys +- Database passwords in plain text +- JWT secrets +- SSH private keys +- OAuth secrets +- Slack webhooks + +### Medium (10 secrets) +Requires some parsing or contextual understanding: +- Base64 encoded AWS key +- Hex-encoded tokens +- Split strings concatenated at runtime +- URL-encoded passwords +- Multi-line private keys in YAML +- Secrets with Unicode characters +- Secrets in SQL/shell comments +- Deprecated config formats + +### Hard (10 secrets) +Well hidden, may challenge even advanced tools: +- ROT13 encoded secrets +- Binary string representations +- Character array joins +- Reversed strings +- Template string constructs +- Secrets in regex patterns +- XOR encrypted values +- Escaped JSON within strings +- Heredoc patterns +- Intentional typos corrected programmatically + +## Usage + +Run secret detection tools against this directory and compare results to the ground truth file (located in `backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json`) to calculate: + +- **Precision**: TP / (TP + FP) - How many detected secrets are real? +- **Recall**: TP / (TP + FN) - How many real secrets were found? +- **F1 Score**: 2 Ɨ (Precision Ɨ Recall) / (Precision + Recall) + +### Expected Performance + +| Tool Type | Expected Easy | Expected Medium | Expected Hard | Total Expected | +|-----------|---------------|-----------------|---------------|----------------| +| Pattern-based (Gitleaks) | 12/12 (100%) | 6-8/10 (60-80%) | 2-4/10 (20-40%) | 20-24/32 | +| Entropy-based (TruffleHog) | 12/12 (100%) | 5-7/10 (50-70%) | 1-3/10 (10-30%) | 18-22/32 | +| LLM-based | 12/12 (100%) | 8-10/10 (80-100%) | 4-8/10 (40-80%) | 24-30/32 | + +## Validation + +Use the validation script to check tool performance: + +```bash +python validate_ground_truth.py --tool-output results.json +``` + +This will calculate precision, recall, and F1 score against the ground truth. diff --git a/test_projects/secret_detection_benchmark/config/app.properties b/test_projects/secret_detection_benchmark/config/app.properties new file mode 100644 index 0000000..d9b2ece --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/app.properties @@ -0,0 +1,9 @@ +# Application properties file +app.name=SecretDetectionBenchmark +app.version=1.0.0 + +# EASY SECRET #8: API Key +api.key=sk_test_4eC39HqLyjWDarjtT1zdp7dc +api.endpoint=https://api.example.com + +logging.level=INFO diff --git a/test_projects/secret_detection_benchmark/config/database.yaml b/test_projects/secret_detection_benchmark/config/database.yaml new file mode 100644 index 0000000..d211c6d --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/database.yaml @@ -0,0 +1,10 @@ +# Database configuration +databases: + production: + host: prod-db.example.com + port: 5432 + # EASY SECRET #6: Azure connection string + connection_string: "DefaultEndpointsProtocol=https;AccountName=prodstore;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;EndpointSuffix=core.windows.net" + staging: + host: staging-db.example.com + port: 5432 diff --git a/test_projects/secret_detection_benchmark/config/keys.yaml b/test_projects/secret_detection_benchmark/config/keys.yaml new file mode 100644 index 0000000..90d1009 --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/keys.yaml @@ -0,0 +1,12 @@ +# Keys configuration +api_keys: + production: + # MEDIUM SECRET #16: Multi-line private key in YAML literal block + private_key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAyLqJZvd5CZxJhLZYLFCqLV9G5k8dFz1LoNwPPfK3qE1k8H4y + FQwNyX3WJZNmKJLOPQMfHZQxGhHJPwZYjKQPYHJ1234567890abcdefghijklmno + -----END RSA PRIVATE KEY----- + + # MEDIUM SECRET #17: Secret with Unicode characters + api_token_intl: "tĆøkęn_Å›Ä™Ć§rėt_įŗƒĆÆth_Å­Ć±ĆÆĆ§Å‘dė_123456" diff --git a/test_projects/secret_detection_benchmark/config/legacy.ini b/test_projects/secret_detection_benchmark/config/legacy.ini new file mode 100644 index 0000000..7a479ec --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/legacy.ini @@ -0,0 +1,8 @@ +[database] +; MEDIUM SECRET #19: Secret in deprecated INI format +password = L3g@cy_DB_P@ssw0rd_2023 + +[api] +; MEDIUM SECRET #20: Commented backup API key +; old_api_key = backup_key_xyz789abc123def456ghi +endpoint = https://api.legacy.example.com diff --git a/test_projects/secret_detection_benchmark/config/oauth.json b/test_projects/secret_detection_benchmark/config/oauth.json new file mode 100644 index 0000000..7fb676b --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/oauth.json @@ -0,0 +1,11 @@ +{ + "oauth_provider": "google", + "client_id": "123456789012-abcdefghijklmnopqrstuvwxyz123456.apps.googleusercontent.com", + "client_secret": "GOCSPX-Ab12Cd34Ef56Gh78Ij90Kl12", + "redirect_uri": "https://example.com/oauth/callback", + "scopes": [ + "openid", + "email", + "profile" + ] +} diff --git a/test_projects/secret_detection_benchmark/config/settings.py b/test_projects/secret_detection_benchmark/config/settings.py new file mode 100644 index 0000000..1f5986a --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/settings.py @@ -0,0 +1,21 @@ +""" +Application settings and configuration +""" + +# EASY SECRET #2: GitHub Personal Access Token +GITHUB_TOKEN = "ghp_vR8jK2mN4pQ6tX9bC3wY7zA1eF5hI8kL" + +# EASY SECRET #3: Stripe API key +STRIPE_SECRET_KEY = "sk_live_51MabcdefghijklmnopqrstuvwxyzABCDEF123456789" + +# Application settings +DEBUG = False +LOG_LEVEL = "INFO" + +# EASY SECRET #4: Database password +DATABASE_CONFIG = { + "host": "prod-db.example.com", + "port": 5432, + "username": "admin", + "password": "ProdDB_P@ssw0rd_2024_Secure!" +} diff --git a/test_projects/secret_detection_benchmark/id_rsa b/test_projects/secret_detection_benchmark/id_rsa new file mode 100644 index 0000000..4f7c8b0 --- /dev/null +++ b/test_projects/secret_detection_benchmark/id_rsa @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAyLqJZvd5CZxJhLZYLFCqLV9G5k8dFz1LoNwPPfK3qE1k8H4yFQwN +yX3WJZNmKJLOPQMfHZQxGhHJPwZYjKQPYHJ1oNwPPfK3qE1k8H4yFQwNyX3WJZNmKJLO +PQMfHZQxGhHJPwZYjKQPYHJ1aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa== +-----END OPENSSH PRIVATE KEY----- diff --git a/test_projects/secret_detection_benchmark/scripts/deploy.sh b/test_projects/secret_detection_benchmark/scripts/deploy.sh new file mode 100644 index 0000000..48c8eca --- /dev/null +++ b/test_projects/secret_detection_benchmark/scripts/deploy.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Deployment script + +# MEDIUM SECRET #14: Secret in environment variable export +export SECRET_API_KEY="sk_prod_1234567890abcdefghijklmnopqrstuvwxyz" + +echo "Deploying application..." + +# MEDIUM SECRET #15: URL-encoded secret in connection string (backup comment) +# backup_connection="mysql://admin:MyP%40ssw0rd%21@db.example.com:3306/prod" + +deploy_app() { + echo "Deployment complete" +} + +deploy_app diff --git a/test_projects/secret_detection_benchmark/scripts/webhook.js b/test_projects/secret_detection_benchmark/scripts/webhook.js new file mode 100644 index 0000000..aeba3f0 --- /dev/null +++ b/test_projects/secret_detection_benchmark/scripts/webhook.js @@ -0,0 +1,13 @@ +// Webhook configuration and handlers + +// EASY SECRET #7: Slack webhook URL +const SLACK_WEBHOOK = "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXX"; + +function sendSlackNotification(message) { + fetch(SLACK_WEBHOOK, { + method: 'POST', + body: JSON.stringify({ text: message }) + }); +} + +module.exports = { sendSlackNotification }; diff --git a/test_projects/secret_detection_benchmark/src/Crypto.go b/test_projects/secret_detection_benchmark/src/Crypto.go new file mode 100644 index 0000000..14d2266 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/Crypto.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "strings" +) + +// HARD SECRET #29: Heredoc with unusual delimiter +const ConfigTemplate = ` +SECRET_KEY=golang_heredoc_secret_999 +END_OF_CONFIG +` + +// HARD SECRET #30: Secret with intentional typo corrected programmatically +const API_KEY_TYPO = "strippe_sk_live_corrected_key" + +func CorrectTypo(s string) string { + return strings.Replace(s, "strippe", "stripe", 1) +} + +func main() { + fmt.Println("Crypto utilities initialized") + correctedKey := CorrectTypo(API_KEY_TYPO) + fmt.Println("Key ready:", correctedKey[:10]+"...") +} diff --git a/test_projects/secret_detection_benchmark/src/Main.java b/test_projects/secret_detection_benchmark/src/Main.java new file mode 100644 index 0000000..9a90be7 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/Main.java @@ -0,0 +1,10 @@ +package com.example.benchmark; + +public class Main { + // EASY SECRET #10: Google OAuth secret in Java + private static final String GOOGLE_OAUTH_SECRET = "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"; + + public static void main(String[] args) { + System.out.println("Application starting..."); + } +} diff --git a/test_projects/secret_detection_benchmark/src/advanced.js b/test_projects/secret_detection_benchmark/src/advanced.js new file mode 100644 index 0000000..6404ce2 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/advanced.js @@ -0,0 +1,19 @@ +// Advanced obfuscation techniques + +// HARD SECRET #25: Template string with escaping +const SECRET_TEMPLATE = `sk_${"prod"}_${"template"}_${"key"}_xyz`; + +// HARD SECRET #26: Secret in regex pattern +const PASSWORD_REGEX = /password_regex_secret_789/; + +// HARD SECRET #27: XORed secret (XOR with key 42) +const XOR_SECRET = [65,82,90,75,94,91,92,75,93,67,65,90,67,92,75,91,67,95]; + +function decodeXOR() { + return String.fromCharCode(...XOR_SECRET.map(c => c ^ 42)); +} + +// HARD SECRET #28: Escaped JSON within string +const CONFIG_JSON = "{\"api_key\":\"sk_escaped_json_key_456\"}"; + +module.exports = { SECRET_TEMPLATE, decodeXOR }; diff --git a/test_projects/secret_detection_benchmark/src/app.py b/test_projects/secret_detection_benchmark/src/app.py new file mode 100644 index 0000000..5d17ade --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/app.py @@ -0,0 +1,19 @@ +""" +Main application entry point +""" +import os + +# EASY SECRET #5: JWT Secret +JWT_SECRET_KEY = "my-super-secret-jwt-key-do-not-share-2024" + +def init_app(): + """Initialize the application""" + app_config = { + "name": "SecretDetectionBenchmark", + "version": "1.0.0" + } + return app_config + +if __name__ == "__main__": + print("Application starting...") + init_app() diff --git a/test_projects/secret_detection_benchmark/src/config.py b/test_projects/secret_detection_benchmark/src/config.py new file mode 100644 index 0000000..22ab19a --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/config.py @@ -0,0 +1,19 @@ +""" +Configuration with moderately obfuscated secrets +""" +import base64 + +# MEDIUM SECRET #11: Base64 encoded AWS key +AWS_KEY_ENCODED = "QUtJQUlPU0ZPRE5ON0VYQU1QTEU=" + +# MEDIUM SECRET #12: Hex-encoded API token +HEX_TOKEN = "6170695f746f6b656e5f616263313233787977373839" + +# MEDIUM SECRET #13: Split secret concatenated at runtime +DB_PASS_PART1 = "MySecure" +DB_PASS_PART2 = "Password" +DB_PASS_PART3 = "2024!" +DATABASE_PASSWORD = DB_PASS_PART1 + DB_PASS_PART2 + DB_PASS_PART3 + +def get_aws_key(): + return base64.b64decode(AWS_KEY_ENCODED).decode() diff --git a/test_projects/secret_detection_benchmark/src/database.sql b/test_projects/secret_detection_benchmark/src/database.sql new file mode 100644 index 0000000..e43eac7 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/database.sql @@ -0,0 +1,15 @@ +-- Database initialization script + +CREATE DATABASE prod_db; + +-- MEDIUM SECRET #18: Secret in SQL comment +-- Connection string: postgresql://admin:Pr0dDB_S3cr3t_P@ss@db.prod.example.com:5432/prod_db + +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + username VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL +); + +-- Insert test data +INSERT INTO users (username, email) VALUES ('admin', 'admin@example.com'); diff --git a/test_projects/secret_detection_benchmark/src/obfuscated.py b/test_projects/secret_detection_benchmark/src/obfuscated.py new file mode 100644 index 0000000..ead77d4 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/obfuscated.py @@ -0,0 +1,23 @@ +""" +Heavily obfuscated secrets - hard to detect +""" +import codecs + +# HARD SECRET #21: ROT13 encoded secret +SECRET_ROT13 = "fx_yvir_frperg_xrl_12345" + +# HARD SECRET #22: Binary string representation +GITHUB_TOKEN_BYTES = b'\x67\x68\x70\x5f\x4d\x79\x47\x69\x74\x48\x75\x62\x54\x6f\x6b\x65\x6e\x31\x32\x33\x34\x35\x36' + +# HARD SECRET #23: Character array join +AWS_SECRET_CHARS = ['A','W','S','_','S','E','C','R','E','T','_','K','E','Y','_','X','Y','Z','7','8','9'] +AWS_SECRET = ''.join(AWS_SECRET_CHARS) + +# HARD SECRET #24: Reversed string that's un-reversed at runtime +TOKEN_REVERSED = "321cba_desrever_nekot_ipa" + +def get_rot13_secret(): + return codecs.decode(SECRET_ROT13, 'rot_13') + +def get_token(): + return TOKEN_REVERSED[::-1] diff --git a/test_projects/secret_detection_benchmark/validate_ground_truth.py b/test_projects/secret_detection_benchmark/validate_ground_truth.py new file mode 100644 index 0000000..958e21c --- /dev/null +++ b/test_projects/secret_detection_benchmark/validate_ground_truth.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +""" +Validate secret detection tool results against ground truth +""" +import json +import argparse +from pathlib import Path +from typing import Set, Tuple + +def load_ground_truth(ground_truth_file: Path) -> Set[Tuple[str, int]]: + """Load ground truth secrets as set of (file, line) tuples""" + with open(ground_truth_file) as f: + data = json.load(f) + + secrets = set() + for secret in data["secrets"]: + secrets.add((secret["file"], secret["line"])) + + return secrets + +def load_tool_results(results_file: Path) -> Set[Tuple[str, int]]: + """Load tool results as set of (file, line) tuples""" + with open(results_file) as f: + data = json.load(f) + + findings = set() + # Assume SARIF format or custom format with findings_by_file + if "findings_by_file" in data: + for file_path, lines in data["findings_by_file"].items(): + for line in lines: + findings.add((file_path, line)) + + return findings + +def calculate_metrics(ground_truth: Set, detected: Set): + """Calculate precision, recall, and F1 score""" + tp = len(ground_truth & detected) # True positives + fp = len(detected - ground_truth) # False positives + fn = len(ground_truth - detected) # False negatives + + precision = tp / (tp + fp) if (tp + fp) > 0 else 0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0 + f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 + + return { + "true_positives": tp, + "false_positives": fp, + "false_negatives": fn, + "precision": precision * 100, + "recall": recall * 100, + "f1_score": f1 * 100 + } + +def main(): + parser = argparse.ArgumentParser(description="Validate tool results against ground truth") + parser.add_argument("--tool-output", required=True, help="Path to tool output JSON") + parser.add_argument("--ground-truth", + default="../../backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json", + help="Path to ground truth file") + args = parser.parse_args() + + ground_truth = load_ground_truth(Path(args.ground_truth)) + detected = load_tool_results(Path(args.tool_output)) + metrics = calculate_metrics(ground_truth, detected) + + print("\n" + "="*60) + print("Secret Detection Validation Results") + print("="*60) + print(f"Ground Truth Secrets: {len(ground_truth)}") + print(f"Detected Secrets: {len(detected)}") + print(f"\nTrue Positives: {metrics['true_positives']}") + print(f"False Positives: {metrics['false_positives']}") + print(f"False Negatives: {metrics['false_negatives']}") + print(f"\n{'Precision:':<15} {metrics['precision']:.2f}%") + print(f"{'Recall:':<15} {metrics['recall']:.2f}%") + print(f"{'F1 Score:':<15} {metrics['f1_score']:.2f}%") + print("="*60 + "\n") + +if __name__ == "__main__": + main() diff --git a/test_projects/vulnerable_app/README.md b/test_projects/vulnerable_app/README.md index 7603b86..de0920a 100644 --- a/test_projects/vulnerable_app/README.md +++ b/test_projects/vulnerable_app/README.md @@ -82,7 +82,6 @@ curl -X POST "http://localhost:8000/workflows/security_assessment/submit" \ -H "Content-Type: application/json" \ -d '{ "target_path": "/path/to/test_projects/vulnerable_app", - "volume_mode": "ro", "parameters": { "scanner_config": {"check_sensitive": true}, "analyzer_config": {"check_secrets": true, "check_sql": true} diff --git a/test_projects/vulnerable_app/app.py b/test_projects/vulnerable_app/app.py index f9c2e23..655451a 100644 --- a/test_projects/vulnerable_app/app.py +++ b/test_projects/vulnerable_app/app.py @@ -15,7 +15,6 @@ Test vulnerable application for FuzzForge security scanning. Contains intentional security vulnerabilities for testing purposes. """ -import os import subprocess import sqlite3 diff --git a/volumes/env/.env.example b/volumes/env/.env.example new file mode 100644 index 0000000..4be30b9 --- /dev/null +++ b/volumes/env/.env.example @@ -0,0 +1,17 @@ +# FuzzForge Agent Configuration +# Copy this to .env and configure your API keys + +# LiteLLM Model Configuration +LITELLM_MODEL=gemini/gemini-2.0-flash-001 +# LITELLM_PROVIDER=gemini + +# API Keys (uncomment and configure as needed) +# GOOGLE_API_KEY= +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= +# OPENROUTER_API_KEY= +# MISTRAL_API_KEY= + +# Agent Configuration +# DEFAULT_TIMEOUT=120 +# DEFAULT_CONTEXT_ID=default diff --git a/volumes/env/README.md b/volumes/env/README.md new file mode 100644 index 0000000..c53f184 --- /dev/null +++ b/volumes/env/README.md @@ -0,0 +1,22 @@ +# FuzzForge Environment Configuration + +This directory contains environment files that are mounted into Docker containers. + +## Files + +- `.env.example` - Template configuration file +- `.env` - Your actual configuration (create by copying .env.example) + +## Usage + +1. Copy the example file: + ```bash + cp .env.example .env + ``` + +2. Edit `.env` and add your API keys + +3. Restart Docker containers to apply changes: + ```bash + docker-compose restart + ``` diff --git a/workers/README.md b/workers/README.md new file mode 100644 index 0000000..200dae4 --- /dev/null +++ b/workers/README.md @@ -0,0 +1,353 @@ +# FuzzForge Vertical Workers + +This directory contains vertical-specific worker implementations for the Temporal architecture. + +## Architecture + +Each vertical worker is a long-lived container pre-built with domain-specific security toolchains: + +``` +workers/ +ā”œā”€ā”€ rust/ # Rust/Native security (AFL++, cargo-fuzz, gdb, valgrind) +ā”œā”€ā”€ android/ # Android security (apktool, Frida, jadx, MobSF) +ā”œā”€ā”€ web/ # Web security (OWASP ZAP, semgrep, eslint) +ā”œā”€ā”€ ios/ # iOS security (class-dump, Clutch, Frida) +ā”œā”€ā”€ blockchain/ # Smart contract security (mythril, slither, echidna) +ā””ā”€ā”€ go/ # Go security (go-fuzz, staticcheck, gosec) +``` + +## How It Works + +1. **Worker Startup**: Worker discovers workflows from `/app/toolbox/workflows` +2. **Filtering**: Only loads workflows where `metadata.yaml` has `vertical: ` +3. **Dynamic Import**: Dynamically imports workflow Python modules +4. **Registration**: Registers discovered workflows with Temporal +5. **Processing**: Polls Temporal task queue for work + +## Adding a New Vertical + +### Step 1: Create Worker Directory + +```bash +mkdir -p workers/my_vertical +cd workers/my_vertical +``` + +### Step 2: Create Dockerfile + +```dockerfile +# workers/my_vertical/Dockerfile +FROM python:3.11-slim + +# Install your vertical-specific tools +RUN apt-get update && apt-get install -y \ + tool1 \ + tool2 \ + tool3 \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +COPY requirements.txt /tmp/ +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Copy worker files +COPY worker.py /app/worker.py +COPY activities.py /app/activities.py + +WORKDIR /app +ENV PYTHONPATH="/app:/app/toolbox:${PYTHONPATH}" +ENV PYTHONUNBUFFERED=1 + +CMD ["python", "worker.py"] +``` + +### Step 3: Copy Worker Files + +```bash +# Copy from rust worker as template +cp workers/rust/worker.py workers/my_vertical/ +cp workers/rust/activities.py workers/my_vertical/ +cp workers/rust/requirements.txt workers/my_vertical/ +``` + +**Note**: The worker.py and activities.py are generic and work for all verticals. You only need to customize the Dockerfile with your tools. + +### Step 4: Add to docker-compose.yml + +Add profiles to prevent auto-start: + +```yaml +worker-my-vertical: + build: + context: ./workers/my_vertical + dockerfile: Dockerfile + container_name: fuzzforge-worker-my-vertical + profiles: # ā† Prevents auto-start (saves RAM) + - workers + - my_vertical + depends_on: + temporal: + condition: service_healthy + minio: + condition: service_healthy + environment: + TEMPORAL_ADDRESS: temporal:7233 + WORKER_VERTICAL: my_vertical # ā† Important: matches metadata.yaml + WORKER_TASK_QUEUE: my-vertical-queue + MAX_CONCURRENT_ACTIVITIES: 5 + # MinIO configuration (same for all workers) + STORAGE_BACKEND: s3 + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: fuzzforge + S3_SECRET_KEY: fuzzforge123 + S3_BUCKET: targets + CACHE_DIR: /cache + volumes: + - ./backend/toolbox:/app/toolbox:ro + - worker_my_vertical_cache:/cache + networks: + - fuzzforge-network + restart: "no" # ā† Don't auto-restart +``` + +**Why profiles?** Workers are pre-built but don't auto-start, saving ~1-2GB RAM per worker when idle. + +### Step 5: Add Volume + +```yaml +volumes: + worker_my_vertical_cache: + name: fuzzforge_worker_my_vertical_cache +``` + +### Step 6: Create Workflows for Your Vertical + +```bash +mkdir -p backend/toolbox/workflows/my_workflow +``` + +**metadata.yaml:** +```yaml +name: my_workflow +version: 1.0.0 +vertical: my_vertical # ā† Must match WORKER_VERTICAL +``` + +**workflow.py:** +```python +from temporalio import workflow +from datetime import timedelta + +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self, target_id: str) -> dict: + # Download target + target_path = await workflow.execute_activity( + "get_target", + target_id, + start_to_close_timeout=timedelta(minutes=5) + ) + + # Your analysis logic here + results = {"status": "success"} + + # Cleanup + await workflow.execute_activity( + "cleanup_cache", + target_path, + start_to_close_timeout=timedelta(minutes=1) + ) + + return results +``` + +### Step 7: Test + +```bash +# Start services +docker-compose -f docker-compose.temporal.yaml up -d + +# Check worker logs +docker logs -f fuzzforge-worker-my-vertical + +# You should see: +# "Discovered workflow: MyWorkflow from my_workflow (vertical: my_vertical)" +``` + +## Worker Components + +### worker.py + +Generic worker entrypoint. Handles: +- Workflow discovery from mounted `/app/toolbox` +- Dynamic import of workflow modules +- Connection to Temporal +- Task queue polling + +**No customization needed** - works for all verticals. + +### activities.py + +Common activities available to all workflows: + +- `get_target(target_id: str) -> str`: Download target from MinIO +- `cleanup_cache(target_path: str) -> None`: Remove cached target +- `upload_results(workflow_id, results, format) -> str`: Upload results to MinIO + +**Can be extended** with vertical-specific activities: + +```python +# workers/my_vertical/activities.py + +from temporalio import activity + +@activity.defn(name="my_custom_activity") +async def my_custom_activity(input_data: str) -> str: + # Your vertical-specific logic + return "result" + +# Add to worker.py activities list: +# activities=[..., my_custom_activity] +``` + +### Dockerfile + +**Only component that needs customization** for each vertical. Install your tools here. + +## Configuration + +### Environment Variables + +All workers support these environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `TEMPORAL_ADDRESS` | `localhost:7233` | Temporal server address | +| `TEMPORAL_NAMESPACE` | `default` | Temporal namespace | +| `WORKER_VERTICAL` | `rust` | Vertical name (must match metadata.yaml) | +| `WORKER_TASK_QUEUE` | `{vertical}-queue` | Task queue name | +| `MAX_CONCURRENT_ACTIVITIES` | `5` | Max concurrent activities per worker | +| `S3_ENDPOINT` | `http://minio:9000` | MinIO/S3 endpoint | +| `S3_ACCESS_KEY` | `fuzzforge` | S3 access key | +| `S3_SECRET_KEY` | `fuzzforge123` | S3 secret key | +| `S3_BUCKET` | `targets` | Bucket for uploaded targets | +| `CACHE_DIR` | `/cache` | Local cache directory | +| `CACHE_MAX_SIZE` | `10GB` | Max cache size (not enforced yet) | +| `LOG_LEVEL` | `INFO` | Logging level | + +## Scaling + +### Vertical Scaling (More Work Per Worker) + +Increase concurrent activities: + +```yaml +environment: + MAX_CONCURRENT_ACTIVITIES: 10 # Handle 10 tasks at once +``` + +### Horizontal Scaling (More Workers) + +```bash +# Scale to 3 workers for rust vertical +docker-compose -f docker-compose.temporal.yaml up -d --scale worker-rust=3 + +# Each worker polls the same task queue +# Temporal automatically load balances +``` + +## Troubleshooting + +### Worker Not Discovering Workflows + +Check: +1. Volume mount is correct: `./backend/toolbox:/app/toolbox:ro` +2. Workflow has `metadata.yaml` with correct `vertical:` field +3. Workflow has `workflow.py` with `@workflow.defn` decorated class +4. Worker logs show discovery attempt + +### Cannot Connect to Temporal + +Check: +1. Temporal container is healthy: `docker ps` +2. Network connectivity: `docker exec worker-rust ping temporal` +3. `TEMPORAL_ADDRESS` environment variable is correct + +### Cannot Download from MinIO + +Check: +1. MinIO is healthy: `docker ps` +2. Buckets exist: `docker exec fuzzforge-minio mc ls fuzzforge/targets` +3. S3 credentials are correct +4. Target was uploaded: Check MinIO console at http://localhost:9001 + +### Activity Timeouts + +Increase timeout in workflow: + +```python +await workflow.execute_activity( + "my_activity", + args, + start_to_close_timeout=timedelta(hours=2) # Increase from default +) +``` + +## Best Practices + +1. **Keep Dockerfiles lean**: Only install necessary tools +2. **Use multi-stage builds**: Reduce final image size +3. **Pin tool versions**: Ensure reproducibility +4. **Log liberally**: Helps debugging workflow issues +5. **Handle errors gracefully**: Don't fail workflow for non-critical issues +6. **Test locally first**: Use docker-compose before deploying + +## On-Demand Worker Management + +Workers use Docker Compose profiles and CLI-managed lifecycle for resource optimization. + +### How It Works + +1. **Build Time**: `docker-compose build` creates all worker images +2. **Startup**: Workers DON'T auto-start with `docker-compose up -d` +3. **On Demand**: CLI starts workers automatically when workflows need them +4. **Shutdown**: Optional auto-stop after workflow completion + +### Manual Control + +```bash +# Start specific worker +docker start fuzzforge-worker-ossfuzz + +# Stop specific worker +docker stop fuzzforge-worker-ossfuzz + +# Check worker status +docker ps --filter "name=fuzzforge-worker" +``` + +### CLI Auto-Management + +```bash +# Auto-start enabled by default +ff workflow run ossfuzz_campaign . project_name=zlib + +# Disable auto-start +ff workflow run ossfuzz_campaign . project_name=zlib --no-auto-start + +# Auto-stop after completion +ff workflow run ossfuzz_campaign . project_name=zlib --wait --auto-stop +``` + +### Resource Savings + +- **Before**: All workers running = ~8GB RAM +- **After**: Only core services running = ~1.2GB RAM +- **Savings**: ~6-7GB RAM when idle + +## Examples + +See existing verticals for examples: +- `workers/rust/` - Complete working example +- `backend/toolbox/workflows/rust_test/` - Simple test workflow diff --git a/workers/android/Dockerfile b/workers/android/Dockerfile new file mode 100644 index 0000000..a3bb9d4 --- /dev/null +++ b/workers/android/Dockerfile @@ -0,0 +1,94 @@ +# FuzzForge Vertical Worker: Android Security +# +# Pre-installed tools for Android security analysis: +# - Android SDK (adb, aapt) +# - apktool (APK decompilation) +# - jadx (Dex to Java decompiler) +# - Frida (dynamic instrumentation) +# - androguard (Python APK analysis) +# - MobSF dependencies + +FROM python:3.11-slim-bookworm + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + # Build essentials + build-essential \ + git \ + curl \ + wget \ + unzip \ + # Java (required for Android tools) + openjdk-17-jdk \ + # Android tools dependencies + lib32stdc++6 \ + lib32z1 \ + # Frida dependencies + libc6-dev \ + # XML/Binary analysis + libxml2-dev \ + libxslt-dev \ + # Network tools + netcat-openbsd \ + tcpdump \ + # Cleanup + && rm -rf /var/lib/apt/lists/* + +# Install Android SDK Command Line Tools +ENV ANDROID_HOME=/opt/android-sdk +ENV PATH="${ANDROID_HOME}/cmdline-tools/latest/bin:${ANDROID_HOME}/platform-tools:${PATH}" + +RUN mkdir -p ${ANDROID_HOME}/cmdline-tools && \ + cd ${ANDROID_HOME}/cmdline-tools && \ + wget -q https://dl.google.com/android/repository/commandlinetools-linux-9477386_latest.zip && \ + unzip -q commandlinetools-linux-9477386_latest.zip && \ + mv cmdline-tools latest && \ + rm commandlinetools-linux-9477386_latest.zip && \ + # Accept licenses + yes | ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --licenses && \ + # Install platform tools (adb, fastboot) + ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager "platform-tools" "build-tools;33.0.0" + +# Install apktool +RUN wget -q https://raw.githubusercontent.com/iBotPeaches/Apktool/master/scripts/linux/apktool -O /usr/local/bin/apktool && \ + wget -q https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.9.3.jar -O /usr/local/bin/apktool.jar && \ + chmod +x /usr/local/bin/apktool + +# Install jadx (Dex to Java decompiler) +RUN wget -q https://github.com/skylot/jadx/releases/download/v1.4.7/jadx-1.4.7.zip -O /tmp/jadx.zip && \ + unzip -q /tmp/jadx.zip -d /opt/jadx && \ + ln -s /opt/jadx/bin/jadx /usr/local/bin/jadx && \ + ln -s /opt/jadx/bin/jadx-gui /usr/local/bin/jadx-gui && \ + rm /tmp/jadx.zip + +# Install Python dependencies for Android security tools +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + +# Install androguard (Python APK analysis framework) +RUN pip3 install --no-cache-dir androguard pyaxmlparser + +# Install Frida +RUN pip3 install --no-cache-dir frida-tools frida + +# Create cache directory +RUN mkdir -p /cache && chmod 755 /cache + +# Copy worker entrypoint (generic, works for all verticals) +COPY worker.py /app/worker.py + +# Add toolbox to Python path (mounted at runtime) +ENV PYTHONPATH="/app:/app/toolbox:${PYTHONPATH}" +ENV PYTHONUNBUFFERED=1 +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 + +# Healthcheck +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD python3 -c "import sys; sys.exit(0)" + +# Run worker +CMD ["python3", "/app/worker.py"] diff --git a/workers/android/requirements.txt b/workers/android/requirements.txt new file mode 100644 index 0000000..3cbd013 --- /dev/null +++ b/workers/android/requirements.txt @@ -0,0 +1,19 @@ +# Temporal Python SDK +temporalio>=1.5.0 + +# S3/MinIO client +boto3>=1.34.0 +botocore>=1.34.0 + +# Data validation +pydantic>=2.5.0 + +# YAML parsing +PyYAML>=6.0.1 + +# Utilities +python-dotenv>=1.0.0 +aiofiles>=23.2.1 + +# Logging +structlog>=24.1.0 diff --git a/workers/android/worker.py b/workers/android/worker.py new file mode 100644 index 0000000..1254ab5 --- /dev/null +++ b/workers/android/worker.py @@ -0,0 +1,309 @@ +""" +FuzzForge Vertical Worker: Rust/Native Security + +This worker: +1. Discovers workflows for the 'rust' vertical from mounted toolbox +2. Dynamically imports and registers workflow classes +3. Connects to Temporal and processes tasks +4. Handles activities for target download/upload from MinIO +""" + +import asyncio +import importlib +import inspect +import logging +import os +import sys +from pathlib import Path +from typing import List, Any + +import yaml +from temporalio.client import Client +from temporalio.worker import Worker + +# Add toolbox to path for workflow and activity imports +sys.path.insert(0, '/app/toolbox') + +# Import common storage activities +from toolbox.common.storage_activities import ( + get_target_activity, + cleanup_cache_activity, + upload_results_activity +) + +# Configure logging +logging.basicConfig( + level=os.getenv('LOG_LEVEL', 'INFO'), + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +async def discover_workflows(vertical: str) -> List[Any]: + """ + Discover workflows for this vertical from mounted toolbox. + + Args: + vertical: The vertical name (e.g., 'rust', 'android', 'web') + + Returns: + List of workflow classes decorated with @workflow.defn + """ + workflows = [] + toolbox_path = Path("/app/toolbox/workflows") + + if not toolbox_path.exists(): + logger.warning(f"Toolbox path does not exist: {toolbox_path}") + return workflows + + logger.info(f"Scanning for workflows in: {toolbox_path}") + + for workflow_dir in toolbox_path.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + metadata_file = workflow_dir / "metadata.yaml" + if not metadata_file.exists(): + logger.debug(f"No metadata.yaml in {workflow_dir.name}, skipping") + continue + + try: + # Parse metadata + with open(metadata_file) as f: + metadata = yaml.safe_load(f) + + # Check if workflow is for this vertical + workflow_vertical = metadata.get("vertical") + if workflow_vertical != vertical: + logger.debug( + f"Workflow {workflow_dir.name} is for vertical '{workflow_vertical}', " + f"not '{vertical}', skipping" + ) + continue + + # Check if workflow.py exists + workflow_file = workflow_dir / "workflow.py" + if not workflow_file.exists(): + logger.warning( + f"Workflow {workflow_dir.name} has metadata but no workflow.py, skipping" + ) + continue + + # Dynamically import workflow module + module_name = f"toolbox.workflows.{workflow_dir.name}.workflow" + logger.info(f"Importing workflow module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import workflow module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @workflow.defn decorated classes + found_workflows = False + for name, obj in inspect.getmembers(module, inspect.isclass): + # Check if class has Temporal workflow definition + if hasattr(obj, '__temporal_workflow_definition'): + workflows.append(obj) + found_workflows = True + logger.info( + f"āœ“ Discovered workflow: {name} from {workflow_dir.name} " + f"(vertical: {vertical})" + ) + + if not found_workflows: + logger.warning( + f"Workflow {workflow_dir.name} has no @workflow.defn decorated classes" + ) + + except Exception as e: + logger.error( + f"Error processing workflow {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(workflows)} workflows for vertical '{vertical}'") + return workflows + + +async def discover_activities(workflows_dir: Path) -> List[Any]: + """ + Discover activities from workflow directories. + + Looks for activities.py files alongside workflow.py in each workflow directory. + + Args: + workflows_dir: Path to workflows directory + + Returns: + List of activity functions decorated with @activity.defn + """ + activities = [] + + if not workflows_dir.exists(): + logger.warning(f"Workflows directory does not exist: {workflows_dir}") + return activities + + logger.info(f"Scanning for workflow activities in: {workflows_dir}") + + for workflow_dir in workflows_dir.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + # Check if activities.py exists + activities_file = workflow_dir / "activities.py" + if not activities_file.exists(): + logger.debug(f"No activities.py in {workflow_dir.name}, skipping") + continue + + try: + # Dynamically import activities module + module_name = f"toolbox.workflows.{workflow_dir.name}.activities" + logger.info(f"Importing activities module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import activities module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @activity.defn decorated functions + found_activities = False + for name, obj in inspect.getmembers(module, inspect.isfunction): + # Check if function has Temporal activity definition + if hasattr(obj, '__temporal_activity_definition'): + activities.append(obj) + found_activities = True + logger.info( + f"āœ“ Discovered activity: {name} from {workflow_dir.name}" + ) + + if not found_activities: + logger.warning( + f"Workflow {workflow_dir.name} has activities.py but no @activity.defn decorated functions" + ) + + except Exception as e: + logger.error( + f"Error processing activities from {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(activities)} workflow-specific activities") + return activities + + +async def main(): + """Main worker entry point""" + # Get configuration from environment + vertical = os.getenv("WORKER_VERTICAL", "rust") + temporal_address = os.getenv("TEMPORAL_ADDRESS", "localhost:7233") + temporal_namespace = os.getenv("TEMPORAL_NAMESPACE", "default") + task_queue = os.getenv("WORKER_TASK_QUEUE", f"{vertical}-queue") + max_concurrent_activities = int(os.getenv("MAX_CONCURRENT_ACTIVITIES", "5")) + + logger.info("=" * 60) + logger.info(f"FuzzForge Vertical Worker: {vertical}") + logger.info("=" * 60) + logger.info(f"Temporal Address: {temporal_address}") + logger.info(f"Temporal Namespace: {temporal_namespace}") + logger.info(f"Task Queue: {task_queue}") + logger.info(f"Max Concurrent Activities: {max_concurrent_activities}") + logger.info("=" * 60) + + # Discover workflows for this vertical + logger.info(f"Discovering workflows for vertical: {vertical}") + workflows = await discover_workflows(vertical) + + if not workflows: + logger.error(f"No workflows found for vertical: {vertical}") + logger.error("Worker cannot start without workflows. Exiting...") + sys.exit(1) + + # Discover activities from workflow directories + logger.info("Discovering workflow-specific activities...") + workflows_dir = Path("/app/toolbox/workflows") + workflow_activities = await discover_activities(workflows_dir) + + # Combine common storage activities with workflow-specific activities + activities = [ + get_target_activity, + cleanup_cache_activity, + upload_results_activity + ] + workflow_activities + + logger.info( + f"Total activities registered: {len(activities)} " + f"(3 common + {len(workflow_activities)} workflow-specific)" + ) + + # Connect to Temporal + logger.info(f"Connecting to Temporal at {temporal_address}...") + try: + client = await Client.connect( + temporal_address, + namespace=temporal_namespace + ) + logger.info("āœ“ Connected to Temporal successfully") + except Exception as e: + logger.error(f"Failed to connect to Temporal: {e}", exc_info=True) + sys.exit(1) + + # Create worker with discovered workflows and activities + logger.info(f"Creating worker on task queue: {task_queue}") + + try: + worker = Worker( + client, + task_queue=task_queue, + workflows=workflows, + activities=activities, + max_concurrent_activities=max_concurrent_activities + ) + logger.info("āœ“ Worker created successfully") + except Exception as e: + logger.error(f"Failed to create worker: {e}", exc_info=True) + sys.exit(1) + + # Start worker + logger.info("=" * 60) + logger.info(f"šŸš€ Worker started for vertical '{vertical}'") + logger.info(f"šŸ“¦ Registered {len(workflows)} workflows") + logger.info(f"āš™ļø Registered {len(activities)} activities") + logger.info(f"šŸ“Ø Listening on task queue: {task_queue}") + logger.info("=" * 60) + logger.info("Worker is ready to process tasks...") + + try: + await worker.run() + except KeyboardInterrupt: + logger.info("Shutting down worker (keyboard interrupt)...") + except Exception as e: + logger.error(f"Worker error: {e}", exc_info=True) + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Worker stopped") + except Exception as e: + logger.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) diff --git a/workers/ossfuzz/Dockerfile b/workers/ossfuzz/Dockerfile new file mode 100644 index 0000000..3d49daf --- /dev/null +++ b/workers/ossfuzz/Dockerfile @@ -0,0 +1,45 @@ +# OSS-Fuzz Worker - Generic fuzzing using OSS-Fuzz infrastructure +FROM gcr.io/oss-fuzz-base/base-builder:latest + +# Install Python, Docker CLI, and dependencies (use Python 3.8 from base image) +RUN apt-get update && apt-get install -y \ + python3-pip \ + python3-dev \ + git \ + docker.io \ + && rm -rf /var/lib/apt/lists/* + +# Upgrade pip +RUN python3 -m pip install --upgrade pip + +# Install Temporal Python SDK and dependencies +RUN pip3 install --no-cache-dir \ + temporalio==1.5.0 \ + boto3==1.34.50 \ + pyyaml==6.0.1 \ + psutil==5.9.8 + +# Create necessary directories +RUN mkdir -p /app /cache /corpus /output + +# Set environment variables +ENV PYTHONPATH=/app +ENV WORKER_VERTICAL=ossfuzz +ENV MAX_CONCURRENT_ACTIVITIES=2 +ENV CACHE_DIR=/cache +ENV CACHE_MAX_SIZE=50GB +ENV CACHE_TTL=30d + +# Clone OSS-Fuzz repo (will be cached in /cache by worker) +# This is just to have helper scripts available +RUN git clone --depth=1 https://github.com/google/oss-fuzz.git /opt/oss-fuzz + +# Copy worker code +COPY worker.py /app/ +COPY activities.py /app/ +COPY requirements.txt /app/ + +WORKDIR /app + +# Run worker +CMD ["python3", "worker.py"] diff --git a/workers/ossfuzz/activities.py b/workers/ossfuzz/activities.py new file mode 100644 index 0000000..7b0ef7c --- /dev/null +++ b/workers/ossfuzz/activities.py @@ -0,0 +1,413 @@ +""" +OSS-Fuzz Campaign Activities + +Activities for running OSS-Fuzz campaigns using Google's infrastructure. +""" + +import logging +import os +import subprocess +import shutil +from pathlib import Path +from typing import Dict, Any, List, Optional +from datetime import datetime + +import yaml +from temporalio import activity + +logger = logging.getLogger(__name__) + +# Paths +OSS_FUZZ_REPO = Path("/opt/oss-fuzz") +CACHE_DIR = Path(os.getenv("CACHE_DIR", "/cache")) + + +@activity.defn(name="load_ossfuzz_project") +async def load_ossfuzz_project_activity(project_name: str) -> Dict[str, Any]: + """ + Load OSS-Fuzz project configuration from project.yaml. + + Args: + project_name: Name of the OSS-Fuzz project (e.g., "curl", "sqlite3") + + Returns: + Dictionary with project config, paths, and metadata + """ + logger.info(f"Loading OSS-Fuzz project: {project_name}") + + # Update OSS-Fuzz repo if it exists, clone if not + if OSS_FUZZ_REPO.exists(): + logger.info("Updating OSS-Fuzz repository...") + subprocess.run( + ["git", "-C", str(OSS_FUZZ_REPO), "pull", "--depth=1"], + check=False # Don't fail if already up to date + ) + else: + logger.info("Cloning OSS-Fuzz repository...") + subprocess.run( + [ + "git", "clone", "--depth=1", + "https://github.com/google/oss-fuzz.git", + str(OSS_FUZZ_REPO) + ], + check=True + ) + + # Find project directory + project_path = OSS_FUZZ_REPO / "projects" / project_name + if not project_path.exists(): + raise ValueError( + f"Project '{project_name}' not found in OSS-Fuzz. " + f"Available projects: https://github.com/google/oss-fuzz/tree/master/projects" + ) + + # Read project.yaml + config_file = project_path / "project.yaml" + if not config_file.exists(): + raise ValueError(f"No project.yaml found for project '{project_name}'") + + with open(config_file) as f: + config = yaml.safe_load(f) + + # Add paths + config["project_name"] = project_name + config["project_path"] = str(project_path) + config["dockerfile_path"] = str(project_path / "Dockerfile") + config["build_script_path"] = str(project_path / "build.sh") + + # Validate required fields + if not config.get("language"): + logger.warning(f"No language specified in project.yaml for {project_name}") + + logger.info( + f"āœ“ Loaded project {project_name}: " + f"language={config.get('language', 'unknown')}, " + f"engines={config.get('fuzzing_engines', [])}, " + f"sanitizers={config.get('sanitizers', [])}" + ) + + return config + + +@activity.defn(name="build_ossfuzz_project") +async def build_ossfuzz_project_activity( + project_name: str, + project_config: Dict[str, Any], + sanitizer: Optional[str] = None, + engine: Optional[str] = None +) -> Dict[str, Any]: + """ + Build OSS-Fuzz project directly using build.sh (no Docker-in-Docker). + + Args: + project_name: Name of the project + project_config: Configuration from project.yaml + sanitizer: Override sanitizer (default: first from project.yaml) + engine: Override engine (default: first from project.yaml) + + Returns: + Dictionary with build results and discovered fuzz targets + """ + logger.info(f"Building OSS-Fuzz project: {project_name}") + + # Determine sanitizer and engine + sanitizers = project_config.get("sanitizers", ["address"]) + engines = project_config.get("fuzzing_engines", ["libfuzzer"]) + + use_sanitizer = sanitizer if sanitizer else sanitizers[0] + use_engine = engine if engine else engines[0] + + logger.info(f"Building with sanitizer={use_sanitizer}, engine={use_engine}") + + # Setup directories + src_dir = Path("/src") + out_dir = Path("/out") + src_dir.mkdir(exist_ok=True) + out_dir.mkdir(exist_ok=True) + + # Clean previous build artifacts + for item in out_dir.glob("*"): + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + + # Copy project files from OSS-Fuzz repo to /src + project_path = Path(project_config["project_path"]) + build_script = project_path / "build.sh" + + if not build_script.exists(): + raise Exception(f"build.sh not found for project {project_name}") + + logger.info(f"Copying project files from {project_path} to {src_dir}") + + # Copy build.sh + shutil.copy2(build_script, src_dir / "build.sh") + os.chmod(src_dir / "build.sh", 0o755) + + # Copy any fuzzer source files (*.cc, *.c, *.cpp files) + for pattern in ["*.cc", "*.c", "*.cpp", "*.h", "*.hh", "*.hpp"]: + for src_file in project_path.glob(pattern): + dest_file = src_dir / src_file.name + shutil.copy2(src_file, dest_file) + logger.info(f"Copied: {src_file.name}") + + # Clone project source code to subdirectory + main_repo = project_config.get("main_repo") + work_dir = src_dir + + if main_repo: + logger.info(f"Cloning project source from {main_repo}") + project_src_dir = src_dir / project_name + + # Remove existing directory if present + if project_src_dir.exists(): + shutil.rmtree(project_src_dir) + + clone_cmd = ["git", "clone", "--depth=1", main_repo, str(project_src_dir)] + result = subprocess.run(clone_cmd, capture_output=True, text=True, timeout=600) + + if result.returncode != 0: + logger.warning(f"Failed to clone {main_repo}: {result.stderr}") + logger.info("Continuing without cloning (build.sh may download source)") + else: + # Copy build.sh into the project source directory + shutil.copy2(src_dir / "build.sh", project_src_dir / "build.sh") + os.chmod(project_src_dir / "build.sh", 0o755) + # build.sh should run from within the project directory + work_dir = project_src_dir + logger.info(f"Build will run from: {work_dir}") + else: + logger.info("No main_repo in project.yaml, build.sh will download source") + + # Set OSS-Fuzz environment variables + build_env = os.environ.copy() + build_env.update({ + "SRC": str(src_dir), + "OUT": str(out_dir), + "FUZZING_ENGINE": use_engine, + "SANITIZER": use_sanitizer, + "ARCHITECTURE": "x86_64", + # Use clang's built-in libfuzzer instead of separate library + "LIB_FUZZING_ENGINE": "-fsanitize=fuzzer", + }) + + # Set sanitizer flags + if use_sanitizer == "address": + build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fsanitize=address" + build_env["CXXFLAGS"] = build_env.get("CXXFLAGS", "") + " -fsanitize=address" + elif use_sanitizer == "memory": + build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fsanitize=memory" + build_env["CXXFLAGS"] = build_env.get("CXXFLAGS", "") + " -fsanitize=memory" + elif use_sanitizer == "undefined": + build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fsanitize=undefined" + build_env["CXXFLAGS"] = build_env.get("CXXFLAGS", "") + " -fsanitize=undefined" + + # Execute build.sh from the work directory + logger.info(f"Executing build.sh in {work_dir}") + build_cmd = ["bash", "./build.sh"] + + result = subprocess.run( + build_cmd, + cwd=str(work_dir), + env=build_env, + capture_output=True, + text=True, + timeout=1800 # 30 minutes max build time + ) + + if result.returncode != 0: + logger.error(f"Build failed:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}") + raise Exception(f"Build failed for {project_name}: {result.stderr}") + + logger.info("āœ“ Build completed successfully") + logger.info(f"Build output:\n{result.stdout[-2000:]}") # Last 2000 chars + + # Discover fuzz targets in /out + fuzz_targets = [] + for file in out_dir.glob("*"): + if file.is_file() and os.access(file, os.X_OK): + # Check if it's a fuzz target (executable, not .so/.a/.o) + if file.suffix not in ['.so', '.a', '.o', '.zip']: + fuzz_targets.append(str(file)) + logger.info(f"Found fuzz target: {file.name}") + + if not fuzz_targets: + logger.warning(f"No fuzz targets found in {out_dir}") + logger.info(f"Directory contents: {list(out_dir.glob('*'))}") + + return { + "fuzz_targets": fuzz_targets, + "build_log": result.stdout[-5000:], # Last 5000 chars + "sanitizer_used": use_sanitizer, + "engine_used": use_engine, + "out_dir": str(out_dir) + } + + +@activity.defn(name="fuzz_target") +async def fuzz_target_activity( + target_path: str, + engine: str, + duration_seconds: int, + corpus_dir: Optional[str] = None, + dict_file: Optional[str] = None +) -> Dict[str, Any]: + """ + Run fuzzing on a target with specified engine. + + Args: + target_path: Path to fuzz target executable + engine: Fuzzing engine (libfuzzer, afl, honggfuzz) + duration_seconds: How long to fuzz + corpus_dir: Optional corpus directory + dict_file: Optional dictionary file + + Returns: + Dictionary with fuzzing stats and results + """ + logger.info(f"Fuzzing {Path(target_path).name} with {engine} for {duration_seconds}s") + + # Prepare corpus directory + if not corpus_dir: + corpus_dir = str(CACHE_DIR / "corpus" / Path(target_path).stem) + Path(corpus_dir).mkdir(parents=True, exist_ok=True) + + output_dir = CACHE_DIR / "output" / Path(target_path).stem + output_dir.mkdir(parents=True, exist_ok=True) + + start_time = datetime.now() + + try: + if engine == "libfuzzer": + cmd = [ + target_path, + corpus_dir, + f"-max_total_time={duration_seconds}", + "-print_final_stats=1", + f"-artifact_prefix={output_dir}/" + ] + if dict_file: + cmd.append(f"-dict={dict_file}") + + elif engine == "afl": + cmd = [ + "afl-fuzz", + "-i", corpus_dir if Path(corpus_dir).glob("*") else "-", # Empty corpus OK + "-o", str(output_dir), + "-t", "1000", # Timeout per execution + "-m", "none", # No memory limit + "--", target_path, "@@" + ] + + elif engine == "honggfuzz": + cmd = [ + "honggfuzz", + f"--run_time={duration_seconds}", + "-i", corpus_dir, + "-o", str(output_dir), + "--", target_path + ] + + else: + raise ValueError(f"Unsupported fuzzing engine: {engine}") + + logger.info(f"Starting fuzzer: {' '.join(cmd[:5])}...") + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=duration_seconds + 120 # Add 2 minute buffer + ) + + end_time = datetime.now() + elapsed = (end_time - start_time).total_seconds() + + # Parse stats from output + stats = parse_fuzzing_stats(result.stdout, result.stderr, engine) + stats["elapsed_time"] = elapsed + stats["target_name"] = Path(target_path).name + stats["engine"] = engine + + # Find crashes + crashes = find_crashes(output_dir) + stats["crashes"] = len(crashes) + stats["crash_files"] = crashes + + # Collect new corpus files + new_corpus = collect_corpus(corpus_dir) + stats["corpus_size"] = len(new_corpus) + stats["corpus_files"] = new_corpus + + logger.info( + f"āœ“ Fuzzing completed: {stats.get('total_executions', 0)} execs, " + f"{len(crashes)} crashes" + ) + + return stats + + except subprocess.TimeoutExpired: + logger.warning(f"Fuzzing timed out after {duration_seconds}s") + return { + "target_name": Path(target_path).name, + "engine": engine, + "status": "timeout", + "elapsed_time": duration_seconds + } + + +def parse_fuzzing_stats(stdout: str, stderr: str, engine: str) -> Dict[str, Any]: + """Parse fuzzing statistics from output""" + stats = {} + + if engine == "libfuzzer": + # Parse libFuzzer stats + for line in (stdout + stderr).split('\n'): + if "#" in line and "NEW" in line: + # Example: #8192 NEW cov: 1234 ft: 5678 corp: 89/10KB + parts = line.split() + for i, part in enumerate(parts): + if part.startswith("cov:"): + stats["coverage"] = int(parts[i+1]) + elif part.startswith("corp:"): + stats["corpus_entries"] = int(parts[i+1].split('/')[0]) + elif part.startswith("exec/s:"): + stats["executions_per_sec"] = float(parts[i+1]) + elif part.startswith("#"): + stats["total_executions"] = int(part[1:]) + + elif engine == "afl": + # Parse AFL stats (would need to read fuzzer_stats file) + pass + + elif engine == "honggfuzz": + # Parse Honggfuzz stats + pass + + return stats + + +def find_crashes(output_dir: Path) -> List[str]: + """Find crash files in output directory""" + crashes = [] + + # libFuzzer crash files start with "crash-" or "leak-" + for pattern in ["crash-*", "leak-*", "timeout-*"]: + crashes.extend([str(f) for f in output_dir.glob(pattern)]) + + # AFL crashes in crashes/ subdirectory + crashes_dir = output_dir / "crashes" + if crashes_dir.exists(): + crashes.extend([str(f) for f in crashes_dir.glob("*") if f.is_file()]) + + return crashes + + +def collect_corpus(corpus_dir: str) -> List[str]: + """Collect corpus files""" + corpus_path = Path(corpus_dir) + if not corpus_path.exists(): + return [] + + return [str(f) for f in corpus_path.glob("*") if f.is_file()] diff --git a/workers/ossfuzz/requirements.txt b/workers/ossfuzz/requirements.txt new file mode 100644 index 0000000..72ea0aa --- /dev/null +++ b/workers/ossfuzz/requirements.txt @@ -0,0 +1,4 @@ +temporalio==1.5.0 +boto3==1.34.50 +pyyaml==6.0.1 +psutil==5.9.8 diff --git a/workers/ossfuzz/worker.py b/workers/ossfuzz/worker.py new file mode 100644 index 0000000..c92dee0 --- /dev/null +++ b/workers/ossfuzz/worker.py @@ -0,0 +1,319 @@ +""" +FuzzForge Vertical Worker: OSS-Fuzz Campaigns + +This worker: +1. Discovers workflows for the 'ossfuzz' vertical from mounted toolbox +2. Dynamically imports and registers workflow classes +3. Connects to Temporal and processes tasks +4. Handles activities for OSS-Fuzz project building and fuzzing +""" + +import asyncio +import importlib +import inspect +import logging +import os +import sys +from pathlib import Path +from typing import List, Any + +import yaml +from temporalio.client import Client +from temporalio.worker import Worker + +# Add toolbox to path for workflow and activity imports +sys.path.insert(0, '/app/toolbox') + +# Import common storage activities +from toolbox.common.storage_activities import ( + get_target_activity, + cleanup_cache_activity, + upload_results_activity +) + +# Import OSS-Fuzz specific activities +from activities import ( + load_ossfuzz_project_activity, + build_ossfuzz_project_activity, + fuzz_target_activity +) + +# Configure logging +logging.basicConfig( + level=os.getenv('LOG_LEVEL', 'INFO'), + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +async def discover_workflows(vertical: str) -> List[Any]: + """ + Discover workflows for this vertical from mounted toolbox. + + Args: + vertical: The vertical name (e.g., 'ossfuzz') + + Returns: + List of workflow classes decorated with @workflow.defn + """ + workflows = [] + toolbox_path = Path("/app/toolbox/workflows") + + if not toolbox_path.exists(): + logger.warning(f"Toolbox path does not exist: {toolbox_path}") + return workflows + + logger.info(f"Scanning for workflows in: {toolbox_path}") + + for workflow_dir in toolbox_path.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + metadata_file = workflow_dir / "metadata.yaml" + if not metadata_file.exists(): + logger.debug(f"No metadata.yaml in {workflow_dir.name}, skipping") + continue + + try: + # Parse metadata + with open(metadata_file) as f: + metadata = yaml.safe_load(f) + + # Check if workflow is for this vertical + workflow_vertical = metadata.get("vertical") + if workflow_vertical != vertical: + logger.debug( + f"Workflow {workflow_dir.name} is for vertical '{workflow_vertical}', " + f"not '{vertical}', skipping" + ) + continue + + # Check if workflow.py exists + workflow_file = workflow_dir / "workflow.py" + if not workflow_file.exists(): + logger.warning( + f"Workflow {workflow_dir.name} has metadata but no workflow.py, skipping" + ) + continue + + # Dynamically import workflow module + module_name = f"toolbox.workflows.{workflow_dir.name}.workflow" + logger.info(f"Importing workflow module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import workflow module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @workflow.defn decorated classes + found_workflows = False + for name, obj in inspect.getmembers(module, inspect.isclass): + # Check if class has Temporal workflow definition + if hasattr(obj, '__temporal_workflow_definition'): + workflows.append(obj) + found_workflows = True + logger.info( + f"āœ“ Discovered workflow: {name} from {workflow_dir.name} " + f"(vertical: {vertical})" + ) + + if not found_workflows: + logger.warning( + f"Workflow {workflow_dir.name} has no @workflow.defn decorated classes" + ) + + except Exception as e: + logger.error( + f"Error processing workflow {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(workflows)} workflows for vertical '{vertical}'") + return workflows + + +async def discover_activities(workflows_dir: Path) -> List[Any]: + """ + Discover activities from workflow directories. + + Looks for activities.py files alongside workflow.py in each workflow directory. + + Args: + workflows_dir: Path to workflows directory + + Returns: + List of activity functions decorated with @activity.defn + """ + activities = [] + + if not workflows_dir.exists(): + logger.warning(f"Workflows directory does not exist: {workflows_dir}") + return activities + + logger.info(f"Scanning for workflow activities in: {workflows_dir}") + + for workflow_dir in workflows_dir.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + # Check if activities.py exists + activities_file = workflow_dir / "activities.py" + if not activities_file.exists(): + logger.debug(f"No activities.py in {workflow_dir.name}, skipping") + continue + + try: + # Dynamically import activities module + module_name = f"toolbox.workflows.{workflow_dir.name}.activities" + logger.info(f"Importing activities module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import activities module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @activity.defn decorated functions + found_activities = False + for name, obj in inspect.getmembers(module, inspect.isfunction): + # Check if function has Temporal activity definition + if hasattr(obj, '__temporal_activity_definition'): + activities.append(obj) + found_activities = True + logger.info( + f"āœ“ Discovered activity: {name} from {workflow_dir.name}" + ) + + if not found_activities: + logger.warning( + f"Workflow {workflow_dir.name} has activities.py but no @activity.defn decorated functions" + ) + + except Exception as e: + logger.error( + f"Error processing activities from {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(activities)} workflow-specific activities") + return activities + + +async def main(): + """Main worker entry point""" + # Get configuration from environment + vertical = os.getenv("WORKER_VERTICAL", "ossfuzz") + temporal_address = os.getenv("TEMPORAL_ADDRESS", "localhost:7233") + temporal_namespace = os.getenv("TEMPORAL_NAMESPACE", "default") + task_queue = os.getenv("WORKER_TASK_QUEUE", f"{vertical}-queue") + max_concurrent_activities = int(os.getenv("MAX_CONCURRENT_ACTIVITIES", "2")) + + logger.info("=" * 60) + logger.info(f"FuzzForge Vertical Worker: {vertical}") + logger.info("=" * 60) + logger.info(f"Temporal Address: {temporal_address}") + logger.info(f"Temporal Namespace: {temporal_namespace}") + logger.info(f"Task Queue: {task_queue}") + logger.info(f"Max Concurrent Activities: {max_concurrent_activities}") + logger.info("=" * 60) + + # Discover workflows for this vertical + logger.info(f"Discovering workflows for vertical: {vertical}") + workflows = await discover_workflows(vertical) + + if not workflows: + logger.error(f"No workflows found for vertical: {vertical}") + logger.error("Worker cannot start without workflows. Exiting...") + sys.exit(1) + + # Discover activities from workflow directories + logger.info("Discovering workflow-specific activities...") + workflows_dir = Path("/app/toolbox/workflows") + workflow_activities = await discover_activities(workflows_dir) + + # Combine common storage activities, OSS-Fuzz activities, and workflow-specific activities + activities = [ + get_target_activity, + cleanup_cache_activity, + upload_results_activity, + load_ossfuzz_project_activity, + build_ossfuzz_project_activity, + fuzz_target_activity + ] + workflow_activities + + logger.info( + f"Total activities registered: {len(activities)} " + f"(3 common + 3 ossfuzz + {len(workflow_activities)} workflow-specific)" + ) + + # Connect to Temporal + logger.info(f"Connecting to Temporal at {temporal_address}...") + try: + client = await Client.connect( + temporal_address, + namespace=temporal_namespace + ) + logger.info("āœ“ Connected to Temporal successfully") + except Exception as e: + logger.error(f"Failed to connect to Temporal: {e}", exc_info=True) + sys.exit(1) + + # Create worker with discovered workflows and activities + logger.info(f"Creating worker on task queue: {task_queue}") + + try: + worker = Worker( + client, + task_queue=task_queue, + workflows=workflows, + activities=activities, + max_concurrent_activities=max_concurrent_activities + ) + logger.info("āœ“ Worker created successfully") + except Exception as e: + logger.error(f"Failed to create worker: {e}", exc_info=True) + sys.exit(1) + + # Start worker + logger.info("=" * 60) + logger.info(f"šŸš€ Worker started for vertical '{vertical}'") + logger.info(f"šŸ“¦ Registered {len(workflows)} workflows") + logger.info(f"āš™ļø Registered {len(activities)} activities") + logger.info(f"šŸ“Ø Listening on task queue: {task_queue}") + logger.info("=" * 60) + logger.info("Worker is ready to process tasks...") + + try: + await worker.run() + except KeyboardInterrupt: + logger.info("Shutting down worker (keyboard interrupt)...") + except Exception as e: + logger.error(f"Worker error: {e}", exc_info=True) + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Worker stopped") + except Exception as e: + logger.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) diff --git a/workers/python/Dockerfile b/workers/python/Dockerfile new file mode 100644 index 0000000..54a8cb8 --- /dev/null +++ b/workers/python/Dockerfile @@ -0,0 +1,47 @@ +# FuzzForge Vertical Worker: Python Fuzzing +# +# Pre-installed tools for Python fuzzing and security analysis: +# - Python 3.11 +# - Atheris (Python fuzzing) +# - Common Python security tools +# - Temporal worker + +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + # Build essentials for Atheris + build-essential \ + clang \ + llvm \ + # Development tools + git \ + curl \ + wget \ + # Cleanup + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies for Temporal worker +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + +# Create cache directory for downloaded targets +RUN mkdir -p /cache && chmod 755 /cache + +# Copy worker entrypoint +COPY worker.py /app/worker.py + +# Add toolbox and AI module to Python path (mounted at runtime) +ENV PYTHONPATH="/app:/app/toolbox:/app/ai_src:${PYTHONPATH}" +ENV PYTHONUNBUFFERED=1 + +# Healthcheck +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD python3 -c "import sys; sys.exit(0)" + +# Run worker +CMD ["python3", "/app/worker.py"] diff --git a/workers/python/requirements.txt b/workers/python/requirements.txt new file mode 100644 index 0000000..2e30f4a --- /dev/null +++ b/workers/python/requirements.txt @@ -0,0 +1,18 @@ +# Temporal worker dependencies +temporalio>=1.5.0 +pydantic>=2.0.0 + +# Storage (MinIO/S3) +boto3>=1.34.0 + +# Configuration +pyyaml>=6.0.0 + +# HTTP Client (for real-time stats reporting) +httpx>=0.27.0 + +# A2A Agent Communication +a2a-sdk[all]>=0.1.0 + +# Fuzzing +atheris>=2.3.0 diff --git a/workers/python/worker.py b/workers/python/worker.py new file mode 100644 index 0000000..1254ab5 --- /dev/null +++ b/workers/python/worker.py @@ -0,0 +1,309 @@ +""" +FuzzForge Vertical Worker: Rust/Native Security + +This worker: +1. Discovers workflows for the 'rust' vertical from mounted toolbox +2. Dynamically imports and registers workflow classes +3. Connects to Temporal and processes tasks +4. Handles activities for target download/upload from MinIO +""" + +import asyncio +import importlib +import inspect +import logging +import os +import sys +from pathlib import Path +from typing import List, Any + +import yaml +from temporalio.client import Client +from temporalio.worker import Worker + +# Add toolbox to path for workflow and activity imports +sys.path.insert(0, '/app/toolbox') + +# Import common storage activities +from toolbox.common.storage_activities import ( + get_target_activity, + cleanup_cache_activity, + upload_results_activity +) + +# Configure logging +logging.basicConfig( + level=os.getenv('LOG_LEVEL', 'INFO'), + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +async def discover_workflows(vertical: str) -> List[Any]: + """ + Discover workflows for this vertical from mounted toolbox. + + Args: + vertical: The vertical name (e.g., 'rust', 'android', 'web') + + Returns: + List of workflow classes decorated with @workflow.defn + """ + workflows = [] + toolbox_path = Path("/app/toolbox/workflows") + + if not toolbox_path.exists(): + logger.warning(f"Toolbox path does not exist: {toolbox_path}") + return workflows + + logger.info(f"Scanning for workflows in: {toolbox_path}") + + for workflow_dir in toolbox_path.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + metadata_file = workflow_dir / "metadata.yaml" + if not metadata_file.exists(): + logger.debug(f"No metadata.yaml in {workflow_dir.name}, skipping") + continue + + try: + # Parse metadata + with open(metadata_file) as f: + metadata = yaml.safe_load(f) + + # Check if workflow is for this vertical + workflow_vertical = metadata.get("vertical") + if workflow_vertical != vertical: + logger.debug( + f"Workflow {workflow_dir.name} is for vertical '{workflow_vertical}', " + f"not '{vertical}', skipping" + ) + continue + + # Check if workflow.py exists + workflow_file = workflow_dir / "workflow.py" + if not workflow_file.exists(): + logger.warning( + f"Workflow {workflow_dir.name} has metadata but no workflow.py, skipping" + ) + continue + + # Dynamically import workflow module + module_name = f"toolbox.workflows.{workflow_dir.name}.workflow" + logger.info(f"Importing workflow module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import workflow module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @workflow.defn decorated classes + found_workflows = False + for name, obj in inspect.getmembers(module, inspect.isclass): + # Check if class has Temporal workflow definition + if hasattr(obj, '__temporal_workflow_definition'): + workflows.append(obj) + found_workflows = True + logger.info( + f"āœ“ Discovered workflow: {name} from {workflow_dir.name} " + f"(vertical: {vertical})" + ) + + if not found_workflows: + logger.warning( + f"Workflow {workflow_dir.name} has no @workflow.defn decorated classes" + ) + + except Exception as e: + logger.error( + f"Error processing workflow {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(workflows)} workflows for vertical '{vertical}'") + return workflows + + +async def discover_activities(workflows_dir: Path) -> List[Any]: + """ + Discover activities from workflow directories. + + Looks for activities.py files alongside workflow.py in each workflow directory. + + Args: + workflows_dir: Path to workflows directory + + Returns: + List of activity functions decorated with @activity.defn + """ + activities = [] + + if not workflows_dir.exists(): + logger.warning(f"Workflows directory does not exist: {workflows_dir}") + return activities + + logger.info(f"Scanning for workflow activities in: {workflows_dir}") + + for workflow_dir in workflows_dir.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + # Check if activities.py exists + activities_file = workflow_dir / "activities.py" + if not activities_file.exists(): + logger.debug(f"No activities.py in {workflow_dir.name}, skipping") + continue + + try: + # Dynamically import activities module + module_name = f"toolbox.workflows.{workflow_dir.name}.activities" + logger.info(f"Importing activities module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import activities module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @activity.defn decorated functions + found_activities = False + for name, obj in inspect.getmembers(module, inspect.isfunction): + # Check if function has Temporal activity definition + if hasattr(obj, '__temporal_activity_definition'): + activities.append(obj) + found_activities = True + logger.info( + f"āœ“ Discovered activity: {name} from {workflow_dir.name}" + ) + + if not found_activities: + logger.warning( + f"Workflow {workflow_dir.name} has activities.py but no @activity.defn decorated functions" + ) + + except Exception as e: + logger.error( + f"Error processing activities from {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(activities)} workflow-specific activities") + return activities + + +async def main(): + """Main worker entry point""" + # Get configuration from environment + vertical = os.getenv("WORKER_VERTICAL", "rust") + temporal_address = os.getenv("TEMPORAL_ADDRESS", "localhost:7233") + temporal_namespace = os.getenv("TEMPORAL_NAMESPACE", "default") + task_queue = os.getenv("WORKER_TASK_QUEUE", f"{vertical}-queue") + max_concurrent_activities = int(os.getenv("MAX_CONCURRENT_ACTIVITIES", "5")) + + logger.info("=" * 60) + logger.info(f"FuzzForge Vertical Worker: {vertical}") + logger.info("=" * 60) + logger.info(f"Temporal Address: {temporal_address}") + logger.info(f"Temporal Namespace: {temporal_namespace}") + logger.info(f"Task Queue: {task_queue}") + logger.info(f"Max Concurrent Activities: {max_concurrent_activities}") + logger.info("=" * 60) + + # Discover workflows for this vertical + logger.info(f"Discovering workflows for vertical: {vertical}") + workflows = await discover_workflows(vertical) + + if not workflows: + logger.error(f"No workflows found for vertical: {vertical}") + logger.error("Worker cannot start without workflows. Exiting...") + sys.exit(1) + + # Discover activities from workflow directories + logger.info("Discovering workflow-specific activities...") + workflows_dir = Path("/app/toolbox/workflows") + workflow_activities = await discover_activities(workflows_dir) + + # Combine common storage activities with workflow-specific activities + activities = [ + get_target_activity, + cleanup_cache_activity, + upload_results_activity + ] + workflow_activities + + logger.info( + f"Total activities registered: {len(activities)} " + f"(3 common + {len(workflow_activities)} workflow-specific)" + ) + + # Connect to Temporal + logger.info(f"Connecting to Temporal at {temporal_address}...") + try: + client = await Client.connect( + temporal_address, + namespace=temporal_namespace + ) + logger.info("āœ“ Connected to Temporal successfully") + except Exception as e: + logger.error(f"Failed to connect to Temporal: {e}", exc_info=True) + sys.exit(1) + + # Create worker with discovered workflows and activities + logger.info(f"Creating worker on task queue: {task_queue}") + + try: + worker = Worker( + client, + task_queue=task_queue, + workflows=workflows, + activities=activities, + max_concurrent_activities=max_concurrent_activities + ) + logger.info("āœ“ Worker created successfully") + except Exception as e: + logger.error(f"Failed to create worker: {e}", exc_info=True) + sys.exit(1) + + # Start worker + logger.info("=" * 60) + logger.info(f"šŸš€ Worker started for vertical '{vertical}'") + logger.info(f"šŸ“¦ Registered {len(workflows)} workflows") + logger.info(f"āš™ļø Registered {len(activities)} activities") + logger.info(f"šŸ“Ø Listening on task queue: {task_queue}") + logger.info("=" * 60) + logger.info("Worker is ready to process tasks...") + + try: + await worker.run() + except KeyboardInterrupt: + logger.info("Shutting down worker (keyboard interrupt)...") + except Exception as e: + logger.error(f"Worker error: {e}", exc_info=True) + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Worker stopped") + except Exception as e: + logger.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) diff --git a/workers/rust/Dockerfile b/workers/rust/Dockerfile new file mode 100644 index 0000000..ba32679 --- /dev/null +++ b/workers/rust/Dockerfile @@ -0,0 +1,87 @@ +# FuzzForge Vertical Worker: Rust/Native Security +# +# Pre-installed tools for Rust and native binary security analysis: +# - Rust toolchain (rustc, cargo) +# - AFL++ (fuzzing) +# - cargo-fuzz (Rust fuzzing) +# - gdb (debugging) +# - valgrind (memory analysis) +# - AddressSanitizer/MemorySanitizer support +# - Common reverse engineering tools + +FROM rust:1.83-slim-bookworm + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + # Build essentials + build-essential \ + cmake \ + git \ + curl \ + wget \ + pkg-config \ + libssl-dev \ + # AFL++ dependencies + clang \ + llvm \ + # Debugging and analysis tools + gdb \ + valgrind \ + strace \ + # Binary analysis (binutils includes objdump, readelf, etc.) + binutils \ + # Network tools + netcat-openbsd \ + tcpdump \ + # Python for Temporal worker + python3 \ + python3-pip \ + python3-venv \ + # Cleanup + && rm -rf /var/lib/apt/lists/* + +# Install AFL++ +RUN git clone https://github.com/AFLplusplus/AFLplusplus /tmp/aflplusplus && \ + cd /tmp/aflplusplus && \ + make all && \ + make install && \ + cd / && \ + rm -rf /tmp/aflplusplus + +# Install Rust toolchain components (nightly required for cargo-fuzz) +RUN rustup install nightly && \ + rustup default nightly && \ + rustup component add rustfmt clippy && \ + rustup target add x86_64-unknown-linux-musl + +# Install cargo-fuzz and other Rust security tools +RUN cargo install --locked \ + cargo-fuzz \ + cargo-audit \ + cargo-outdated \ + cargo-tree + +# Install Python dependencies for Temporal worker +COPY requirements.txt /tmp/requirements.txt +RUN pip3 install --break-system-packages --no-cache-dir -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + +# Create cache directory for downloaded targets +RUN mkdir -p /cache && chmod 755 /cache + +# Copy worker entrypoint +COPY worker.py /app/worker.py + +# Add toolbox to Python path (mounted at runtime) +ENV PYTHONPATH="/app:/app/toolbox:${PYTHONPATH}" +ENV PYTHONUNBUFFERED=1 + +# Healthcheck +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD python3 -c "import sys; sys.exit(0)" + +# Run worker +CMD ["python3", "/app/worker.py"] diff --git a/workers/rust/requirements.txt b/workers/rust/requirements.txt new file mode 100644 index 0000000..a9ce9f6 --- /dev/null +++ b/workers/rust/requirements.txt @@ -0,0 +1,22 @@ +# Temporal Python SDK +temporalio>=1.5.0 + +# S3/MinIO client +boto3>=1.34.0 +botocore>=1.34.0 + +# Data validation +pydantic>=2.5.0 + +# YAML parsing +PyYAML>=6.0.1 + +# Utilities +python-dotenv>=1.0.0 +aiofiles>=23.2.1 + +# HTTP Client (for real-time stats reporting) +httpx>=0.27.0 + +# Logging +structlog>=24.1.0 diff --git a/workers/rust/worker.py b/workers/rust/worker.py new file mode 100644 index 0000000..1254ab5 --- /dev/null +++ b/workers/rust/worker.py @@ -0,0 +1,309 @@ +""" +FuzzForge Vertical Worker: Rust/Native Security + +This worker: +1. Discovers workflows for the 'rust' vertical from mounted toolbox +2. Dynamically imports and registers workflow classes +3. Connects to Temporal and processes tasks +4. Handles activities for target download/upload from MinIO +""" + +import asyncio +import importlib +import inspect +import logging +import os +import sys +from pathlib import Path +from typing import List, Any + +import yaml +from temporalio.client import Client +from temporalio.worker import Worker + +# Add toolbox to path for workflow and activity imports +sys.path.insert(0, '/app/toolbox') + +# Import common storage activities +from toolbox.common.storage_activities import ( + get_target_activity, + cleanup_cache_activity, + upload_results_activity +) + +# Configure logging +logging.basicConfig( + level=os.getenv('LOG_LEVEL', 'INFO'), + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +async def discover_workflows(vertical: str) -> List[Any]: + """ + Discover workflows for this vertical from mounted toolbox. + + Args: + vertical: The vertical name (e.g., 'rust', 'android', 'web') + + Returns: + List of workflow classes decorated with @workflow.defn + """ + workflows = [] + toolbox_path = Path("/app/toolbox/workflows") + + if not toolbox_path.exists(): + logger.warning(f"Toolbox path does not exist: {toolbox_path}") + return workflows + + logger.info(f"Scanning for workflows in: {toolbox_path}") + + for workflow_dir in toolbox_path.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + metadata_file = workflow_dir / "metadata.yaml" + if not metadata_file.exists(): + logger.debug(f"No metadata.yaml in {workflow_dir.name}, skipping") + continue + + try: + # Parse metadata + with open(metadata_file) as f: + metadata = yaml.safe_load(f) + + # Check if workflow is for this vertical + workflow_vertical = metadata.get("vertical") + if workflow_vertical != vertical: + logger.debug( + f"Workflow {workflow_dir.name} is for vertical '{workflow_vertical}', " + f"not '{vertical}', skipping" + ) + continue + + # Check if workflow.py exists + workflow_file = workflow_dir / "workflow.py" + if not workflow_file.exists(): + logger.warning( + f"Workflow {workflow_dir.name} has metadata but no workflow.py, skipping" + ) + continue + + # Dynamically import workflow module + module_name = f"toolbox.workflows.{workflow_dir.name}.workflow" + logger.info(f"Importing workflow module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import workflow module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @workflow.defn decorated classes + found_workflows = False + for name, obj in inspect.getmembers(module, inspect.isclass): + # Check if class has Temporal workflow definition + if hasattr(obj, '__temporal_workflow_definition'): + workflows.append(obj) + found_workflows = True + logger.info( + f"āœ“ Discovered workflow: {name} from {workflow_dir.name} " + f"(vertical: {vertical})" + ) + + if not found_workflows: + logger.warning( + f"Workflow {workflow_dir.name} has no @workflow.defn decorated classes" + ) + + except Exception as e: + logger.error( + f"Error processing workflow {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(workflows)} workflows for vertical '{vertical}'") + return workflows + + +async def discover_activities(workflows_dir: Path) -> List[Any]: + """ + Discover activities from workflow directories. + + Looks for activities.py files alongside workflow.py in each workflow directory. + + Args: + workflows_dir: Path to workflows directory + + Returns: + List of activity functions decorated with @activity.defn + """ + activities = [] + + if not workflows_dir.exists(): + logger.warning(f"Workflows directory does not exist: {workflows_dir}") + return activities + + logger.info(f"Scanning for workflow activities in: {workflows_dir}") + + for workflow_dir in workflows_dir.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Skip special directories + if workflow_dir.name.startswith('.') or workflow_dir.name == '__pycache__': + continue + + # Check if activities.py exists + activities_file = workflow_dir / "activities.py" + if not activities_file.exists(): + logger.debug(f"No activities.py in {workflow_dir.name}, skipping") + continue + + try: + # Dynamically import activities module + module_name = f"toolbox.workflows.{workflow_dir.name}.activities" + logger.info(f"Importing activities module: {module_name}") + + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.error( + f"Failed to import activities module {module_name}: {e}", + exc_info=True + ) + continue + + # Find @activity.defn decorated functions + found_activities = False + for name, obj in inspect.getmembers(module, inspect.isfunction): + # Check if function has Temporal activity definition + if hasattr(obj, '__temporal_activity_definition'): + activities.append(obj) + found_activities = True + logger.info( + f"āœ“ Discovered activity: {name} from {workflow_dir.name}" + ) + + if not found_activities: + logger.warning( + f"Workflow {workflow_dir.name} has activities.py but no @activity.defn decorated functions" + ) + + except Exception as e: + logger.error( + f"Error processing activities from {workflow_dir.name}: {e}", + exc_info=True + ) + continue + + logger.info(f"Discovered {len(activities)} workflow-specific activities") + return activities + + +async def main(): + """Main worker entry point""" + # Get configuration from environment + vertical = os.getenv("WORKER_VERTICAL", "rust") + temporal_address = os.getenv("TEMPORAL_ADDRESS", "localhost:7233") + temporal_namespace = os.getenv("TEMPORAL_NAMESPACE", "default") + task_queue = os.getenv("WORKER_TASK_QUEUE", f"{vertical}-queue") + max_concurrent_activities = int(os.getenv("MAX_CONCURRENT_ACTIVITIES", "5")) + + logger.info("=" * 60) + logger.info(f"FuzzForge Vertical Worker: {vertical}") + logger.info("=" * 60) + logger.info(f"Temporal Address: {temporal_address}") + logger.info(f"Temporal Namespace: {temporal_namespace}") + logger.info(f"Task Queue: {task_queue}") + logger.info(f"Max Concurrent Activities: {max_concurrent_activities}") + logger.info("=" * 60) + + # Discover workflows for this vertical + logger.info(f"Discovering workflows for vertical: {vertical}") + workflows = await discover_workflows(vertical) + + if not workflows: + logger.error(f"No workflows found for vertical: {vertical}") + logger.error("Worker cannot start without workflows. Exiting...") + sys.exit(1) + + # Discover activities from workflow directories + logger.info("Discovering workflow-specific activities...") + workflows_dir = Path("/app/toolbox/workflows") + workflow_activities = await discover_activities(workflows_dir) + + # Combine common storage activities with workflow-specific activities + activities = [ + get_target_activity, + cleanup_cache_activity, + upload_results_activity + ] + workflow_activities + + logger.info( + f"Total activities registered: {len(activities)} " + f"(3 common + {len(workflow_activities)} workflow-specific)" + ) + + # Connect to Temporal + logger.info(f"Connecting to Temporal at {temporal_address}...") + try: + client = await Client.connect( + temporal_address, + namespace=temporal_namespace + ) + logger.info("āœ“ Connected to Temporal successfully") + except Exception as e: + logger.error(f"Failed to connect to Temporal: {e}", exc_info=True) + sys.exit(1) + + # Create worker with discovered workflows and activities + logger.info(f"Creating worker on task queue: {task_queue}") + + try: + worker = Worker( + client, + task_queue=task_queue, + workflows=workflows, + activities=activities, + max_concurrent_activities=max_concurrent_activities + ) + logger.info("āœ“ Worker created successfully") + except Exception as e: + logger.error(f"Failed to create worker: {e}", exc_info=True) + sys.exit(1) + + # Start worker + logger.info("=" * 60) + logger.info(f"šŸš€ Worker started for vertical '{vertical}'") + logger.info(f"šŸ“¦ Registered {len(workflows)} workflows") + logger.info(f"āš™ļø Registered {len(activities)} activities") + logger.info(f"šŸ“Ø Listening on task queue: {task_queue}") + logger.info("=" * 60) + logger.info("Worker is ready to process tasks...") + + try: + await worker.run() + except KeyboardInterrupt: + logger.info("Shutting down worker (keyboard interrupt)...") + except Exception as e: + logger.error(f"Worker error: {e}", exc_info=True) + raise + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Worker stopped") + except Exception as e: + logger.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1)