commit b46f050aefe8b85c7c2a0e8c70575d9fb1ac422a Author: AFredefon Date: Fri Jan 30 09:57:48 2026 +0100 feat: FuzzForge AI - complete rewrite for OSS release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b3a4049 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.egg-info +*.whl +.env +.mypy_cache +.pytest_cache +.ruff_cache +.venv +.vscode +__pycache__ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..95ed564 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14.2 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6c92c16 --- /dev/null +++ b/Makefile @@ -0,0 +1,93 @@ +.PHONY: help install sync format lint typecheck test build-modules clean + +SHELL := /bin/bash + +# Default target +help: + @echo "FuzzForge OSS Development Commands" + @echo "" + @echo " make install - Install all dependencies" + @echo " make sync - Sync shared packages from upstream" + @echo " make format - Format code with ruff" + @echo " make lint - Lint code with ruff" + @echo " make typecheck - Type check with mypy" + @echo " make test - Run all tests" + @echo " make build-modules - Build all module container images" + @echo " make clean - Clean build artifacts" + @echo "" + +# Install all dependencies +install: + uv sync + +# Sync shared packages from upstream fuzzforge-core +sync: + @if [ -z "$(UPSTREAM)" ]; then \ + echo "Usage: make sync UPSTREAM=/path/to/fuzzforge-core"; \ + exit 1; \ + fi + ./scripts/sync-upstream.sh $(UPSTREAM) + +# Format all packages +format: + @for pkg in packages/fuzzforge-*/; do \ + if [ -f "$$pkg/pyproject.toml" ]; then \ + echo "Formatting $$pkg..."; \ + cd "$$pkg" && uv run ruff format . && cd -; \ + fi \ + done + +# Lint all packages +lint: + @for pkg in packages/fuzzforge-*/; do \ + if [ -f "$$pkg/pyproject.toml" ]; then \ + echo "Linting $$pkg..."; \ + cd "$$pkg" && uv run ruff check . && cd -; \ + fi \ + done + +# Type check all packages +typecheck: + @for pkg in packages/fuzzforge-*/; do \ + if [ -f "$$pkg/pyproject.toml" ] && [ -f "$$pkg/mypy.ini" ]; then \ + echo "Type checking $$pkg..."; \ + cd "$$pkg" && uv run mypy . && cd -; \ + fi \ + done + +# Run all tests +test: + @for pkg in packages/fuzzforge-*/; do \ + if [ -f "$$pkg/pytest.ini" ]; then \ + echo "Testing $$pkg..."; \ + cd "$$pkg" && uv run pytest && cd -; \ + fi \ + done + +# Build all module container images +build-modules: + @echo "Building FuzzForge module images..." + @echo "This uses self-contained storage at ~/.fuzzforge/containers/" + @for module in fuzzforge-modules/*/; do \ + if [ -f "$$module/Dockerfile" ] && \ + [ "$$module" != "fuzzforge-modules/fuzzforge-modules-sdk/" ] && \ + [ "$$module" != "fuzzforge-modules/fuzzforge-module-template/" ]; then \ + name=$$(basename $$module); \ + version=$$(grep 'version' "$$module/pyproject.toml" 2>/dev/null | head -1 | sed 's/.*"\(.*\)".*/\1/' || echo "0.1.0"); \ + echo "Building $$name:$$version..."; \ + podman --root ~/.fuzzforge/containers/storage --runroot ~/.fuzzforge/containers/run \ + build -t "fuzzforge-$$name:$$version" "$$module" || exit 1; \ + fi \ + done + @echo "" + @echo "✓ All modules built successfully!" + @echo " Images stored in: ~/.fuzzforge/containers/storage" + +# Clean build artifacts +clean: + find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + find . -type d -name ".pytest_cache" -exec rm -rf {} + 2>/dev/null || true + find . -type d -name ".mypy_cache" -exec rm -rf {} + 2>/dev/null || true + find . -type d -name ".ruff_cache" -exec rm -rf {} + 2>/dev/null || true + find . -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true + find . -type f -name "*.pyc" -delete 2>/dev/null || true diff --git a/README.md b/README.md new file mode 100644 index 0000000..a119cdc --- /dev/null +++ b/README.md @@ -0,0 +1,283 @@ +

FuzzForge OSS

+

AI-Powered Security Research Orchestration via MCP

+ +

+ Discord + License: Apache 2.0 + Python 3.12+ + MCP Compatible + Website +

+ +

+ Let AI agents orchestrate your security research workflows locally +

+ +

+ + Overview • + Features • + Installation • + Usage Guide • + Modules • + Contributing + +

+ +--- + +> 🚧 **FuzzForge OSS is under active development.** Expect breaking changes and new features! + +--- + +## 🚀 Overview + +**FuzzForge OSS** is an open-source runtime that enables AI agents (GitHub Copilot, Claude, etc.) to orchestrate security research workflows through the **Model Context Protocol (MCP)**. + +### The Core: Modules + +At the heart of FuzzForge are **modules** - containerized security tools that AI agents can discover, configure, and orchestrate. Each module encapsulates a specific security capability (static analysis, fuzzing, crash analysis, etc.) and runs in an isolated container. + +- **🔌 Plug & Play**: Modules are self-contained - just pull and run +- **🤖 AI-Native**: Designed for AI agent orchestration via MCP +- **🔗 Composable**: Chain modules together into automated workflows +- **📦 Extensible**: Build custom modules with the Python SDK + +The OSS runtime handles module discovery, execution, and result collection. Security modules (developed separately) provide the actual security tooling - from static analyzers to fuzzers to crash triagers. + +Instead of manually running security tools, describe what you want and let your AI assistant handle it. + +### 🎬 Use Case: Rust Fuzzing Pipeline + +> **Scenario**: Fuzz a Rust crate to discover vulnerabilities using AI-assisted harness generation and parallel fuzzing. + + + + + + + + + + + + + + +
1️⃣ Analyze, Generate & Validate Harnesses2️⃣ Run Parallel Continuous Fuzzing
FuzzForge Demo - Analysis PipelineFuzzForge Demo - Parallel Fuzzing
AI agent analyzes code, generates harnesses, and validates they compileMultiple fuzzing sessions run in parallel with live metrics
+ +--- + +## ⭐ Support the Project + +If you find FuzzForge useful, please **star the repo** to support development! 🚀 + + + GitHub Stars + + +--- + +## ✨ Features + +| Feature | Description | +|---------|-------------| +| 🤖 **AI-Native** | Built for MCP - works with GitHub Copilot, Claude, and any MCP-compatible agent | +| 📦 **Containerized** | Each module runs in isolation via Podman or Docker | +| 🔄 **Continuous Mode** | Long-running tasks (fuzzing) with real-time metrics streaming | +| 🔗 **Workflows** | Chain multiple modules together in automated pipelines | +| 🛠️ **Extensible** | Create custom modules with the Python SDK | +| 🏠 **Local First** | All execution happens on your machine - no cloud required | +| 🔒 **Secure** | Sandboxed containers with no network access by default | + +--- + +## 🏗️ Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ AI Agent (Copilot/Claude) │ +└───────────────────────────┬─────────────────────────────────────┘ + │ MCP Protocol (stdio) + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ FuzzForge MCP Server │ +│ ┌─────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ +│ │list_modules │ │execute_module│ │start_continuous_module │ │ +│ └─────────────┘ └──────────────┘ └────────────────────────┘ │ +└───────────────────────────┬─────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ FuzzForge Runner │ +│ Container Engine (Podman/Docker) │ +└───────────────────────────┬─────────────────────────────────────┘ + │ + ┌───────────────────┼───────────────────┐ + ▼ ▼ ▼ +┌───────────────┐ ┌───────────────┐ ┌───────────────┐ +│ Module A │ │ Module B │ │ Module C │ +│ (Container) │ │ (Container) │ │ (Container) │ +└───────────────┘ └───────────────┘ └───────────────┘ +``` + +--- + +## 📦 Installation + +### Prerequisites + +- **Python 3.12+** +- **[uv](https://docs.astral.sh/uv/)** package manager +- **Podman** (recommended) or Docker + +### Quick Install + +```bash +# Clone the repository +git clone https://github.com/FuzzingLabs/fuzzforge-oss.git +cd fuzzforge-oss + +# Install dependencies +uv sync + +# Start Podman socket (Linux) +systemctl --user start podman.socket +``` + +### Configure MCP for Your AI Agent + +```bash +# For GitHub Copilot +uv run fuzzforge mcp install copilot + +# For Claude Code (CLI) +uv run fuzzforge mcp install claude-code + +# For Claude Desktop (standalone app) +uv run fuzzforge mcp install claude-desktop + +# Verify installation +uv run fuzzforge mcp status +``` + +**Restart your editor** and your AI agent will have access to FuzzForge tools! + +--- + +## 📦 Modules + +FuzzForge modules are containerized security tools that AI agents can orchestrate. The module ecosystem is designed around a simple principle: **the OSS runtime orchestrates, enterprise modules execute**. + +### Module Ecosystem + +| | FuzzForge OSS | FuzzForge Enterprise Modules | +|---|---|---| +| **What** | Runtime & MCP server | Security research modules | +| **License** | Apache 2.0 | BSL 1.1 (Business Source License) | +| **Compatibility** | ✅ Runs any compatible module | ✅ Works with OSS runtime | + +**Enterprise modules** are developed separately and provide production-ready security tooling: + +| Category | Modules | Description | +|----------|---------|-------------| +| 🔍 **Static Analysis** | Rust Analyzer, Solidity Analyzer, Cairo Analyzer | Code analysis and fuzzable function detection | +| 🎯 **Fuzzing** | Cargo Fuzzer, Honggfuzz, AFL++ | Coverage-guided fuzz testing | +| 💥 **Crash Analysis** | Crash Triager, Root Cause Analyzer | Automated crash deduplication and analysis | +| 🔐 **Vulnerability Detection** | Pattern Matcher, Taint Analyzer | Security vulnerability scanning | +| 📝 **Reporting** | Report Generator, SARIF Exporter | Automated security report generation | + +> 💡 **Build your own modules!** The FuzzForge SDK allows you to create custom modules that integrate seamlessly with the OSS runtime. See [Creating Custom Modules](#-creating-custom-modules). + +### Execution Modes + +Modules run in two execution modes: + +#### One-shot Execution + +Run a module once and get results: + +```python +result = execute_module("my-analyzer", assets_path="/path/to/project") +``` + +#### Continuous Execution + +For long-running tasks like fuzzing, with real-time metrics: + +```python +# Start continuous execution +session = start_continuous_module("my-fuzzer", + assets_path="/path/to/project", + configuration={"target": "my_target"}) + +# Check status with live metrics +status = get_continuous_status(session["session_id"]) + +# Stop and collect results +stop_continuous_module(session["session_id"]) +``` + +--- + +## 🛠️ Creating Custom Modules + +Build your own security modules with the FuzzForge SDK: + +```python +from fuzzforge_modules_sdk import FuzzForgeModule, FuzzForgeModuleResults + +class MySecurityModule(FuzzForgeModule): + def _run(self, resources): + self.emit_event("started", target=resources[0].path) + + # Your analysis logic here + results = self.analyze(resources) + + self.emit_progress(100, status="completed", + message=f"Analysis complete") + return FuzzForgeModuleResults.SUCCESS +``` + +📖 See the [Module SDK Guide](fuzzforge-modules/fuzzforge-modules-sdk/README.md) for details. + +--- + +## 📁 Project Structure + +``` +fuzzforge-oss/ +├── fuzzforge-cli/ # Command-line interface +├── fuzzforge-common/ # Shared abstractions (containers, storage) +├── fuzzforge-mcp/ # MCP server for AI agents +├── fuzzforge-modules/ # Security modules +│ └── fuzzforge-modules-sdk/ # Module development SDK +├── fuzzforge-runner/ # Local execution engine +├── fuzzforge-types/ # Type definitions & schemas +└── demo/ # Demo projects for testing +``` + +--- + +## 🤝 Contributing + +We welcome contributions from the community! + +- 🐛 Report bugs via [GitHub Issues](../../issues) +- 💡 Suggest features or improvements +- 🔧 Submit pull requests +- 📦 Share your custom modules + +See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. + +--- + +## 📄 License + +Apache 2.0 - See [LICENSE](LICENSE) for details. + +--- + +

+ Built with ❤️ by FuzzingLabs +

\ No newline at end of file diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 0000000..68cfca0 --- /dev/null +++ b/USAGE.md @@ -0,0 +1,439 @@ +# FuzzForge OSS Usage Guide + +This guide covers everything you need to know to get started with FuzzForge OSS - from installation to running your first security research workflow with AI. + +> **FuzzForge is designed to be used with AI agents** (GitHub Copilot, Claude, etc.) via MCP. +> The CLI is available for advanced users but the primary experience is through natural language interaction with your AI assistant. + +--- + +## Table of Contents + +- [Quick Start](#quick-start) +- [Prerequisites](#prerequisites) +- [Installation](#installation) +- [Building Modules](#building-modules) +- [MCP Server Configuration](#mcp-server-configuration) + - [GitHub Copilot](#github-copilot) + - [Claude Code (CLI)](#claude-code-cli) + - [Claude Desktop](#claude-desktop) +- [Using FuzzForge with AI](#using-fuzzforge-with-ai) +- [CLI Reference](#cli-reference) +- [Environment Variables](#environment-variables) +- [Troubleshooting](#troubleshooting) + +--- + +## Quick Start + +> **Prerequisites:** You need [uv](https://docs.astral.sh/uv/) and [Podman](https://podman.io/) installed. +> See the [Prerequisites](#prerequisites) section for installation instructions. + +```bash +# 1. Clone and install +git clone https://github.com/FuzzingLabs/fuzzforge-oss.git +cd fuzzforge-oss +uv sync + +# 2. Build the module images (one-time setup) +make build-modules + +# 3. Install MCP for your AI agent +uv run fuzzforge mcp install copilot # For VS Code + GitHub Copilot +# OR +uv run fuzzforge mcp install claude-code # For Claude Code CLI + +# 4. Restart your AI agent (VS Code, Claude, etc.) + +# 5. Start talking to your AI: +# "List available FuzzForge modules" +# "Analyze this Rust crate for fuzzable functions" +# "Start fuzzing the parse_input function" +``` + +> **Note:** FuzzForge uses self-contained container storage (`~/.fuzzforge/containers/`) +> which works automatically - no need to configure Podman sockets manually. + +--- + +## Prerequisites + +Before installing FuzzForge OSS, ensure you have: + +- **Python 3.12+** - [Download Python](https://www.python.org/downloads/) +- **uv** package manager - [Install uv](https://docs.astral.sh/uv/) +- **Podman** - Container runtime (Docker also works but Podman is recommended) + +### Installing uv + +```bash +# Linux/macOS +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Or with pip +pip install uv +``` + +### Installing Podman (Linux) + +```bash +# Ubuntu/Debian +sudo apt update && sudo apt install -y podman + +# Fedora/RHEL +sudo dnf install -y podman + +# Arch Linux +sudo pacman -S podman +``` + +--- + +## Installation + +### 1. Clone the Repository + +```bash +git clone https://github.com/FuzzingLabs/fuzzforge-oss.git +cd fuzzforge-oss +``` + +### 2. Install Dependencies + +```bash +uv sync +``` + +This installs all FuzzForge components in a virtual environment. + +### 3. Verify Installation + +```bash +uv run fuzzforge --help +``` + +--- + +## Building Modules + +FuzzForge modules are containerized security tools. After cloning, you need to build them once: + +### Build All Modules + +```bash +# From the fuzzforge-oss directory +make build-modules +``` + +This builds all available modules: +- `fuzzforge-rust-analyzer` - Analyzes Rust code for fuzzable functions +- `fuzzforge-cargo-fuzzer` - Runs cargo-fuzz on Rust crates +- `fuzzforge-harness-validator` - Validates generated fuzzing harnesses +- `fuzzforge-crash-analyzer` - Analyzes crash inputs + +### Build a Single Module + +```bash +# Build a specific module +cd fuzzforge-modules/rust-analyzer +make build +``` + +### Verify Modules are Built + +```bash +# List built module images +podman images | grep fuzzforge +``` + +You should see something like: +``` +fuzzforge-rust-analyzer 0.1.0 abc123def456 2 minutes ago 850 MB +fuzzforge-cargo-fuzzer 0.1.0 789ghi012jkl 2 minutes ago 1.2 GB +... +``` + +--- + +## MCP Server Configuration + +FuzzForge integrates with AI agents through the Model Context Protocol (MCP). Configure your preferred AI agent to use FuzzForge tools. + +### GitHub Copilot + +```bash +# That's it! Just run this command: +uv run fuzzforge mcp install copilot +``` + +The command auto-detects everything: +- **FuzzForge root** - Where FuzzForge is installed +- **Modules path** - Defaults to `fuzzforge-oss/fuzzforge-modules` +- **Podman socket** - Auto-detects `/run/user//podman/podman.sock` + +**Optional overrides** (usually not needed): +```bash +uv run fuzzforge mcp install copilot \ + --modules /path/to/modules \ + --engine docker # if using Docker instead of Podman +``` + +**After installation:** +1. Restart VS Code +2. Open GitHub Copilot Chat +3. FuzzForge tools are now available! + +### Claude Code (CLI) + +```bash +uv run fuzzforge mcp install claude-code +``` + +Installs to `~/.claude.json` so FuzzForge tools are available from any directory. + +**After installation:** +1. Run `claude` from any directory +2. FuzzForge tools are now available! + +### Claude Desktop + +```bash +# Automatic installation +uv run fuzzforge mcp install claude-desktop + +# Verify +uv run fuzzforge mcp status +``` + +**After installation:** +1. Restart Claude Desktop +2. FuzzForge tools are now available! + +### Check MCP Status + +```bash +uv run fuzzforge mcp status +``` + +Shows configuration status for all supported AI agents: + +``` +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Agent ┃ Config Path ┃ Status ┃ FuzzForge Configured ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ GitHub Copilot │ ~/.config/Code/User/mcp.json │ ✓ Exists │ ✓ Yes │ +│ Claude Desktop │ ~/.config/Claude/claude_desktop_config... │ Not found │ - │ +│ Claude Code │ ~/.claude.json │ ✓ Exists │ ✓ Yes │ +└──────────────────────┴───────────────────────────────────────────┴──────────────┴─────────────────────────┘ +``` + +### Generate Config Without Installing + +```bash +# Preview the configuration that would be installed +uv run fuzzforge mcp generate copilot +uv run fuzzforge mcp generate claude-desktop +uv run fuzzforge mcp generate claude-code +``` + +### Remove MCP Configuration + +```bash +uv run fuzzforge mcp uninstall copilot +uv run fuzzforge mcp uninstall claude-desktop +uv run fuzzforge mcp uninstall claude-code +``` + +--- + +## Using FuzzForge with AI + +Once MCP is configured, you interact with FuzzForge through natural language with your AI assistant. + +### Example Conversations + +**Discover available tools:** +``` +You: "What FuzzForge modules are available?" +AI: Uses list_modules → "I found 4 modules: rust-analyzer, cargo-fuzzer, + harness-validator, and crash-analyzer..." +``` + +**Analyze code for fuzzing targets:** +``` +You: "Analyze this Rust crate for functions I should fuzz" +AI: Uses execute_module("rust-analyzer") → "I found 3 good fuzzing candidates: + - parse_input() in src/parser.rs - handles untrusted input + - decode_message() in src/codec.rs - complex parsing logic + ..." +``` + +**Generate and validate harnesses:** +``` +You: "Generate a fuzzing harness for the parse_input function" +AI: Creates harness code, then uses execute_module("harness-validator") + → "Here's a harness that compiles successfully..." +``` + +**Run continuous fuzzing:** +``` +You: "Start fuzzing parse_input for 10 minutes" +AI: Uses start_continuous_module("cargo-fuzzer") → "Started fuzzing session abc123" + +You: "How's the fuzzing going?" +AI: Uses get_continuous_status("abc123") → "Running for 5 minutes: + - 150,000 executions + - 2 crashes found + - 45% edge coverage" + +You: "Stop and show me the crashes" +AI: Uses stop_continuous_module("abc123") → "Found 2 unique crashes..." +``` + +### Available MCP Tools + +| Tool | Description | +|------|-------------| +| `list_modules` | List all available security modules | +| `execute_module` | Run a module once and get results | +| `start_continuous_module` | Start a long-running module (e.g., fuzzing) | +| `get_continuous_status` | Check status of a continuous session | +| `stop_continuous_module` | Stop a continuous session | +| `list_continuous_sessions` | List all active sessions | +| `get_execution_results` | Retrieve results from an execution | +| `execute_workflow` | Run a multi-step workflow | + +--- + +## CLI Reference + +> **Note:** The CLI is for advanced users. Most users should interact with FuzzForge through their AI assistant. + +### MCP Commands + +```bash +uv run fuzzforge mcp status # Check configuration status +uv run fuzzforge mcp install # Install MCP config +uv run fuzzforge mcp uninstall # Remove MCP config +uv run fuzzforge mcp generate # Preview config without installing +``` + +### Module Commands + +```bash +uv run fuzzforge modules list # List available modules +uv run fuzzforge modules info # Show module details +uv run fuzzforge modules run --assets . # Run a module +``` + +### Project Commands + +```bash +uv run fuzzforge project init # Initialize a project +uv run fuzzforge project info # Show project info +uv run fuzzforge project executions # List executions +uv run fuzzforge project results # Get execution results +``` + +--- + +## Environment Variables + +Configure FuzzForge using environment variables: + +```bash +# Project paths +export FUZZFORGE_MODULES_PATH=/path/to/modules +export FUZZFORGE_STORAGE_PATH=/path/to/storage + +# Container engine (uses self-contained storage by default) +export FUZZFORGE_ENGINE__TYPE=podman # or docker +export FUZZFORGE_ENGINE__GRAPHROOT=~/.fuzzforge/containers/storage +export FUZZFORGE_ENGINE__RUNROOT=~/.fuzzforge/containers/run +``` + +--- + +## Troubleshooting + +### Podman Socket Not Found + +``` +Error: Could not connect to Podman socket +``` + +**Solution:** +```bash +# Start the Podman socket +systemctl --user start podman.socket + +# Check the socket path +echo /run/user/$(id -u)/podman/podman.sock +``` + +### Permission Denied on Socket + +``` +Error: Permission denied connecting to Podman socket +``` + +**Solution:** +```bash +# Ensure Podman is installed and your user can run containers +podman run --rm hello-world + +# If using system socket, ensure correct permissions +ls -la /run/user/$(id -u)/podman/ +``` + +> **Note:** FuzzForge OSS uses self-contained storage (`~/.fuzzforge/containers/`) by default, +> which avoids most permission issues with the Podman socket. + +### No Modules Found + +``` +No modules found. +``` + +**Solution:** +1. Build the modules first: `make build-modules` +2. Check the modules path: `uv run fuzzforge modules list` +3. Verify images exist: `podman images | grep fuzzforge` + +### MCP Server Not Starting + +Check the MCP configuration: +```bash +uv run fuzzforge mcp status +``` + +Verify the configuration file path exists and contains valid JSON. + +### Module Container Fails to Build + +```bash +# Build module container manually to see errors +cd fuzzforge-modules/ +podman build -t . +``` + +### Check Logs + +FuzzForge stores execution logs in the storage directory: +```bash +ls -la ~/.fuzzforge/storage/// +``` + +--- + +## Next Steps + +- 📖 Read the [Module SDK Guide](fuzzforge-modules/fuzzforge-modules-sdk/README.md) to create custom modules +- 🎬 Check the demos in the [README](README.md) +- 💬 Join our [Discord](https://discord.gg/8XEX33UUwZ) for support + +--- + +

+ Built with ❤️ by FuzzingLabs +

diff --git a/assets/demopart1.gif b/assets/demopart1.gif new file mode 100644 index 0000000..db09ca0 Binary files /dev/null and b/assets/demopart1.gif differ diff --git a/assets/demopart2.gif b/assets/demopart2.gif new file mode 100644 index 0000000..775f52c Binary files /dev/null and b/assets/demopart2.gif differ diff --git a/fuzzforge-cli/Makefile b/fuzzforge-cli/Makefile new file mode 100644 index 0000000..7912847 --- /dev/null +++ b/fuzzforge-cli/Makefile @@ -0,0 +1,37 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit clean cloc format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) $(TESTS) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) $(TESTS) + +pytest: + uv run pytest -vv $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-cli/README.md b/fuzzforge-cli/README.md new file mode 100644 index 0000000..ed59d48 --- /dev/null +++ b/fuzzforge-cli/README.md @@ -0,0 +1,3 @@ +# FuzzForge CLI + +... diff --git a/fuzzforge-cli/mypy.ini b/fuzzforge-cli/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-cli/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-cli/pyproject.toml b/fuzzforge-cli/pyproject.toml new file mode 100644 index 0000000..eabbc38 --- /dev/null +++ b/fuzzforge-cli/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "fuzzforge-cli" +version = "0.0.1" +description = "FuzzForge CLI - Command-line interface for FuzzForge OSS." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-runner==0.0.1", + "fuzzforge-types==0.0.1", + "rich>=14.0.0", + "typer==0.20.1", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +fuzzforge = "fuzzforge_cli.__main__:main" + +[tool.uv.sources] +fuzzforge-runner = { workspace = true } +fuzzforge-types = { workspace = true } diff --git a/fuzzforge-cli/pytest.ini b/fuzzforge-cli/pytest.ini new file mode 100644 index 0000000..2f4c80e --- /dev/null +++ b/fuzzforge-cli/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto diff --git a/fuzzforge-cli/ruff.toml b/fuzzforge-cli/ruff.toml new file mode 100644 index 0000000..678218a --- /dev/null +++ b/fuzzforge-cli/ruff.toml @@ -0,0 +1,15 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-cli/src/fuzzforge_cli/__init__.py b/fuzzforge-cli/src/fuzzforge_cli/__init__.py new file mode 100644 index 0000000..4578657 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/__init__.py @@ -0,0 +1 @@ +"""TODO.""" diff --git a/fuzzforge-cli/src/fuzzforge_cli/__main__.py b/fuzzforge-cli/src/fuzzforge_cli/__main__.py new file mode 100644 index 0000000..25c97c3 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/__main__.py @@ -0,0 +1,12 @@ +"""TODO.""" + +from fuzzforge_cli.application import application + + +def main() -> None: + """TODO.""" + application() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-cli/src/fuzzforge_cli/application.py b/fuzzforge-cli/src/fuzzforge_cli/application.py new file mode 100644 index 0000000..bf427f3 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/application.py @@ -0,0 +1,96 @@ +"""FuzzForge CLI application.""" + +from pathlib import Path +from typing import Annotated + +from fuzzforge_runner import Runner, Settings +from typer import Context as TyperContext +from typer import Option, Typer + +from fuzzforge_cli.commands import mcp, modules, projects +from fuzzforge_cli.context import Context + +application: Typer = Typer( + name="fuzzforge", + help="FuzzForge OSS - Security research orchestration platform.", +) + + +@application.callback() +def main( + project_path: Annotated[ + Path, + Option( + "--project", + "-p", + envvar="FUZZFORGE_PROJECT__DEFAULT_PATH", + help="Path to the FuzzForge project directory.", + ), + ] = Path.cwd(), + modules_path: Annotated[ + Path, + Option( + "--modules", + "-m", + envvar="FUZZFORGE_MODULES_PATH", + help="Path to the modules directory.", + ), + ] = Path.home() / ".fuzzforge" / "modules", + storage_path: Annotated[ + Path, + Option( + "--storage", + envvar="FUZZFORGE_STORAGE__PATH", + help="Path to the storage directory.", + ), + ] = Path.home() / ".fuzzforge" / "storage", + engine_type: Annotated[ + str, + Option( + "--engine", + envvar="FUZZFORGE_ENGINE__TYPE", + help="Container engine type (docker or podman).", + ), + ] = "podman", + engine_socket: Annotated[ + str, + Option( + "--socket", + envvar="FUZZFORGE_ENGINE__SOCKET", + help="Container engine socket path.", + ), + ] = "", + context: TyperContext = None, # type: ignore[assignment] +) -> None: + """FuzzForge OSS - Security research orchestration platform. + + Execute security research modules in isolated containers. + + """ + from fuzzforge_runner.settings import EngineSettings, ProjectSettings, StorageSettings + + settings = Settings( + engine=EngineSettings( + type=engine_type, # type: ignore[arg-type] + socket=engine_socket, + ), + storage=StorageSettings( + path=storage_path, + ), + project=ProjectSettings( + default_path=project_path, + modules_path=modules_path, + ), + ) + + runner = Runner(settings) + + context.obj = Context( + runner=runner, + project_path=project_path, + ) + + +application.add_typer(mcp.application) +application.add_typer(modules.application) +application.add_typer(projects.application) diff --git a/fuzzforge-cli/src/fuzzforge_cli/commands/__init__.py b/fuzzforge-cli/src/fuzzforge_cli/commands/__init__.py new file mode 100644 index 0000000..4578657 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/commands/__init__.py @@ -0,0 +1 @@ +"""TODO.""" diff --git a/fuzzforge-cli/src/fuzzforge_cli/commands/mcp.py b/fuzzforge-cli/src/fuzzforge_cli/commands/mcp.py new file mode 100644 index 0000000..0101beb --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/commands/mcp.py @@ -0,0 +1,527 @@ +"""MCP server configuration commands for FuzzForge CLI. + +This module provides commands for setting up MCP server connections +with various AI agents (VS Code Copilot, Claude Code, etc.). + +""" + +from __future__ import annotations + +import json +import os +import sys +from enum import StrEnum +from pathlib import Path +from typing import Annotated + +from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax +from rich.table import Table +from typer import Argument, Context, Option, Typer + +application: Typer = Typer( + name="mcp", + help="MCP server configuration commands.", +) + + +class AIAgent(StrEnum): + """Supported AI agents.""" + + COPILOT = "copilot" # GitHub Copilot in VS Code + CLAUDE_DESKTOP = "claude-desktop" # Claude Desktop app + CLAUDE_CODE = "claude-code" # Claude Code CLI (terminal) + + +def _get_copilot_mcp_path() -> Path: + """Get the GitHub Copilot MCP configuration file path. + + GitHub Copilot uses VS Code's mcp.json for MCP servers. + + :returns: Path to the mcp.json file. + + """ + if sys.platform == "darwin": + return Path.home() / "Library" / "Application Support" / "Code" / "User" / "mcp.json" + elif sys.platform == "win32": + return Path(os.environ.get("APPDATA", "")) / "Code" / "User" / "mcp.json" + else: # Linux + return Path.home() / ".config" / "Code" / "User" / "mcp.json" + + +def _get_claude_desktop_mcp_path() -> Path: + """Get the Claude Desktop MCP configuration file path. + + :returns: Path to the claude_desktop_config.json file. + + """ + if sys.platform == "darwin": + return Path.home() / "Library" / "Application Support" / "Claude" / "claude_desktop_config.json" + elif sys.platform == "win32": + return Path(os.environ.get("APPDATA", "")) / "Claude" / "claude_desktop_config.json" + else: # Linux + return Path.home() / ".config" / "Claude" / "claude_desktop_config.json" + + +def _get_claude_code_mcp_path(project_path: Path | None = None) -> Path: + """Get the Claude Code MCP configuration file path. + + Claude Code uses .mcp.json in the project root for project-scoped servers. + + :param project_path: Project directory path. If None, uses current directory. + :returns: Path to the .mcp.json file. + + """ + if project_path: + return project_path / ".mcp.json" + return Path.cwd() / ".mcp.json" + + +def _get_claude_code_user_mcp_path() -> Path: + """Get the Claude Code user-scoped MCP configuration file path. + + :returns: Path to ~/.claude.json file. + + """ + return Path.home() / ".claude.json" + + +def _detect_podman_socket() -> str: + """Auto-detect the Podman socket path. + + :returns: Path to the Podman socket. + + """ + uid = os.getuid() + socket_paths = [ + f"/run/user/{uid}/podman/podman.sock", + "/run/podman/podman.sock", + "/var/run/podman/podman.sock", + ] + + for path in socket_paths: + if Path(path).exists(): + return path + + # Default to user socket + return f"/run/user/{uid}/podman/podman.sock" + + +def _detect_docker_socket() -> str: + """Auto-detect the Docker socket path. + + :returns: Path to the Docker socket. + + """ + socket_paths = [ + "/var/run/docker.sock", + Path.home() / ".docker" / "run" / "docker.sock", + ] + + for path in socket_paths: + if Path(path).exists(): + return str(path) + + return "/var/run/docker.sock" + + +def _find_fuzzforge_root() -> Path: + """Find the FuzzForge installation root. + + :returns: Path to fuzzforge-oss directory. + + """ + # Try to find from current file location + current = Path(__file__).resolve() + + # Walk up to find fuzzforge-oss root + for parent in current.parents: + if (parent / "fuzzforge-mcp").is_dir() and (parent / "fuzzforge-runner").is_dir(): + return parent + + # Fall back to cwd + return Path.cwd() + + +def _generate_mcp_config( + fuzzforge_root: Path, + modules_path: Path, + engine_type: str, + engine_socket: str, +) -> dict: + """Generate MCP server configuration. + + :param fuzzforge_root: Path to fuzzforge-oss installation. + :param modules_path: Path to the modules directory. + :param engine_type: Container engine type (podman or docker). + :param engine_socket: Container engine socket path. + :returns: MCP configuration dictionary. + + """ + venv_python = fuzzforge_root / ".venv" / "bin" / "python" + + # Use uv run if no venv, otherwise use venv python directly + if venv_python.exists(): + command = str(venv_python) + args = ["-m", "fuzzforge_mcp"] + else: + command = "uv" + args = ["--directory", str(fuzzforge_root), "run", "fuzzforge-mcp"] + + # Self-contained storage paths for FuzzForge containers + # This isolates FuzzForge from system Podman and avoids snap issues + fuzzforge_home = Path.home() / ".fuzzforge" + graphroot = fuzzforge_home / "containers" / "storage" + runroot = fuzzforge_home / "containers" / "run" + + return { + "type": "stdio", + "command": command, + "args": args, + "cwd": str(fuzzforge_root), + "env": { + "FUZZFORGE_MODULES_PATH": str(modules_path), + "FUZZFORGE_ENGINE__TYPE": engine_type, + "FUZZFORGE_ENGINE__GRAPHROOT": str(graphroot), + "FUZZFORGE_ENGINE__RUNROOT": str(runroot), + }, + } + + +@application.command( + help="Show current MCP configuration status.", + name="status", +) +def status(context: Context) -> None: + """Show MCP configuration status for all supported agents. + + :param context: Typer context. + + """ + console = Console() + + table = Table(title="MCP Configuration Status") + table.add_column("Agent", style="cyan") + table.add_column("Config Path") + table.add_column("Status") + table.add_column("FuzzForge Configured") + + fuzzforge_root = _find_fuzzforge_root() + + agents = [ + ("GitHub Copilot", _get_copilot_mcp_path(), "servers"), + ("Claude Desktop", _get_claude_desktop_mcp_path(), "mcpServers"), + ("Claude Code", _get_claude_code_user_mcp_path(), "mcpServers"), + ] + + for name, config_path, servers_key in agents: + if config_path.exists(): + try: + config = json.loads(config_path.read_text()) + servers = config.get(servers_key, {}) + has_fuzzforge = "fuzzforge" in servers + table.add_row( + name, + str(config_path), + "[green]✓ Exists[/green]", + "[green]✓ Yes[/green]" if has_fuzzforge else "[yellow]✗ No[/yellow]", + ) + except json.JSONDecodeError: + table.add_row( + name, + str(config_path), + "[red]✗ Invalid JSON[/red]", + "[dim]-[/dim]", + ) + else: + table.add_row( + name, + str(config_path), + "[dim]Not found[/dim]", + "[dim]-[/dim]", + ) + + console.print(table) + + # Show detected environment + console.print() + console.print("[bold]Detected Environment:[/bold]") + console.print(f" FuzzForge Root: {_find_fuzzforge_root()}") + console.print(f" Podman Socket: {_detect_podman_socket()}") + console.print(f" Docker Socket: {_detect_docker_socket()}") + + +@application.command( + help="Generate MCP configuration for an AI agent.", + name="generate", +) +def generate( + context: Context, + agent: Annotated[ + AIAgent, + Argument( + help="AI agent to generate config for (copilot, claude-desktop, or claude-code).", + ), + ], + modules_path: Annotated[ + Path | None, + Option( + "--modules", + "-m", + help="Path to the modules directory.", + ), + ] = None, + engine: Annotated[ + str, + Option( + "--engine", + "-e", + help="Container engine (podman or docker).", + ), + ] = "podman", +) -> None: + """Generate MCP configuration and print to stdout. + + :param context: Typer context. + :param agent: Target AI agent. + :param modules_path: Override modules path. + :param engine: Container engine type. + + """ + console = Console() + fuzzforge_root = _find_fuzzforge_root() + + # Use defaults if not specified + resolved_modules = modules_path or (fuzzforge_root / "fuzzforge-modules") + + # Detect socket + if engine == "podman": + socket = _detect_podman_socket() + else: + socket = _detect_docker_socket() + + # Generate config + server_config = _generate_mcp_config( + fuzzforge_root=fuzzforge_root, + modules_path=resolved_modules, + engine_type=engine, + engine_socket=socket, + ) + + # Format based on agent + if agent == AIAgent.COPILOT: + full_config = {"servers": {"fuzzforge": server_config}} + else: # Claude Desktop or Claude Code + full_config = {"mcpServers": {"fuzzforge": server_config}} + + config_json = json.dumps(full_config, indent=4) + + console.print(Panel( + Syntax(config_json, "json", theme="monokai"), + title=f"MCP Configuration for {agent.value}", + )) + + # Show where to save it + if agent == AIAgent.COPILOT: + config_path = _get_copilot_mcp_path() + elif agent == AIAgent.CLAUDE_CODE: + config_path = _get_claude_code_mcp_path(fuzzforge_root) + else: # Claude Desktop + config_path = _get_claude_desktop_mcp_path() + + console.print() + console.print(f"[bold]Save to:[/bold] {config_path}") + console.print() + console.print("[dim]Or run 'fuzzforge mcp install' to install automatically.[/dim]") + + +@application.command( + help="Install MCP configuration for an AI agent.", + name="install", +) +def install( + context: Context, + agent: Annotated[ + AIAgent, + Argument( + help="AI agent to install config for (copilot, claude-desktop, or claude-code).", + ), + ], + modules_path: Annotated[ + Path | None, + Option( + "--modules", + "-m", + help="Path to the modules directory.", + ), + ] = None, + engine: Annotated[ + str, + Option( + "--engine", + "-e", + help="Container engine (podman or docker).", + ), + ] = "podman", + force: Annotated[ + bool, + Option( + "--force", + "-f", + help="Overwrite existing fuzzforge configuration.", + ), + ] = False, +) -> None: + """Install MCP configuration for the specified AI agent. + + This will create or update the MCP configuration file, adding the + fuzzforge server configuration. + + :param context: Typer context. + :param agent: Target AI agent. + :param modules_path: Override modules path. + :param engine: Container engine type. + :param force: Overwrite existing configuration. + + """ + console = Console() + fuzzforge_root = _find_fuzzforge_root() + + # Determine config path + if agent == AIAgent.COPILOT: + config_path = _get_copilot_mcp_path() + servers_key = "servers" + elif agent == AIAgent.CLAUDE_CODE: + config_path = _get_claude_code_user_mcp_path() + servers_key = "mcpServers" + else: # Claude Desktop + config_path = _get_claude_desktop_mcp_path() + servers_key = "mcpServers" + + # Use defaults if not specified + resolved_modules = modules_path or (fuzzforge_root / "fuzzforge-modules") + + # Detect socket + if engine == "podman": + socket = _detect_podman_socket() + else: + socket = _detect_docker_socket() + + # Generate server config + server_config = _generate_mcp_config( + fuzzforge_root=fuzzforge_root, + modules_path=resolved_modules, + engine_type=engine, + engine_socket=socket, + ) + + # Load existing config or create new + if config_path.exists(): + try: + existing_config = json.loads(config_path.read_text()) + except json.JSONDecodeError: + console.print(f"[red]Error: Invalid JSON in {config_path}[/red]") + console.print("[dim]Please fix the file manually or delete it.[/dim]") + raise SystemExit(1) + + # Check if fuzzforge already exists + servers = existing_config.get(servers_key, {}) + if "fuzzforge" in servers and not force: + console.print("[yellow]FuzzForge is already configured.[/yellow]") + console.print("[dim]Use --force to overwrite existing configuration.[/dim]") + raise SystemExit(1) + + # Add/update fuzzforge + if servers_key not in existing_config: + existing_config[servers_key] = {} + existing_config[servers_key]["fuzzforge"] = server_config + + full_config = existing_config + else: + # Create new config + config_path.parent.mkdir(parents=True, exist_ok=True) + full_config = {servers_key: {"fuzzforge": server_config}} + + # Write config + config_path.write_text(json.dumps(full_config, indent=4)) + + console.print(f"[green]✓ Installed FuzzForge MCP configuration for {agent.value}[/green]") + console.print() + console.print(f"[bold]Configuration file:[/bold] {config_path}") + console.print() + console.print("[bold]Settings:[/bold]") + console.print(f" Modules Path: {resolved_modules}") + console.print(f" Engine: {engine}") + console.print(f" Socket: {socket}") + console.print() + + console.print("[bold]Next steps:[/bold]") + if agent == AIAgent.COPILOT: + console.print(" 1. Restart VS Code") + console.print(" 2. Open Copilot Chat and look for FuzzForge tools") + elif agent == AIAgent.CLAUDE_CODE: + console.print(" 1. Run 'claude' from any directory") + console.print(" 2. FuzzForge tools will be available") + else: # Claude Desktop + console.print(" 1. Restart Claude Desktop") + console.print(" 2. The fuzzforge MCP server will be available") + + +@application.command( + help="Remove MCP configuration for an AI agent.", + name="uninstall", +) +def uninstall( + context: Context, + agent: Annotated[ + AIAgent, + Argument( + help="AI agent to remove config from (copilot, claude-desktop, or claude-code).", + ), + ], +) -> None: + """Remove FuzzForge MCP configuration from the specified AI agent. + + :param context: Typer context. + :param agent: Target AI agent. + + """ + console = Console() + fuzzforge_root = _find_fuzzforge_root() + + # Determine config path + if agent == AIAgent.COPILOT: + config_path = _get_copilot_mcp_path() + servers_key = "servers" + elif agent == AIAgent.CLAUDE_CODE: + config_path = _get_claude_code_user_mcp_path() + servers_key = "mcpServers" + else: # Claude Desktop + config_path = _get_claude_desktop_mcp_path() + servers_key = "mcpServers" + + if not config_path.exists(): + console.print(f"[yellow]Configuration file not found: {config_path}[/yellow]") + return + + try: + config = json.loads(config_path.read_text()) + except json.JSONDecodeError: + console.print(f"[red]Error: Invalid JSON in {config_path}[/red]") + raise SystemExit(1) + + servers = config.get(servers_key, {}) + if "fuzzforge" not in servers: + console.print("[yellow]FuzzForge is not configured.[/yellow]") + return + + # Remove fuzzforge + del servers["fuzzforge"] + + # Write back + config_path.write_text(json.dumps(config, indent=4)) + + console.print(f"[green]✓ Removed FuzzForge MCP configuration from {agent.value}[/green]") + console.print() + console.print("[dim]Restart your AI agent for changes to take effect.[/dim]") diff --git a/fuzzforge-cli/src/fuzzforge_cli/commands/modules.py b/fuzzforge-cli/src/fuzzforge_cli/commands/modules.py new file mode 100644 index 0000000..0e43d3b --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/commands/modules.py @@ -0,0 +1,166 @@ +"""Module management commands for FuzzForge CLI.""" + +import asyncio +from pathlib import Path +from typing import Annotated, Any + +from rich.console import Console +from rich.table import Table +from typer import Argument, Context, Option, Typer + +from fuzzforge_cli.context import get_project_path, get_runner + +application: Typer = Typer( + name="modules", + help="Module management commands.", +) + + +@application.command( + help="List available modules.", + name="list", +) +def list_modules( + context: Context, +) -> None: + """List all available modules. + + :param context: Typer context. + + """ + runner = get_runner(context) + modules = runner.list_modules() + + console = Console() + + if not modules: + console.print("[yellow]No modules found.[/yellow]") + console.print(f" Modules directory: {runner.settings.modules_path}") + return + + table = Table(title="Available Modules") + table.add_column("Identifier", style="cyan") + table.add_column("Available") + table.add_column("Description") + + for module in modules: + table.add_row( + module.identifier, + "✓" if module.available else "✗", + module.description or "-", + ) + + console.print(table) + + +@application.command( + help="Execute a module.", + name="run", +) +def run_module( + context: Context, + module_identifier: Annotated[ + str, + Argument( + help="Identifier of the module to execute.", + ), + ], + assets_path: Annotated[ + Path | None, + Option( + "--assets", + "-a", + help="Path to input assets.", + ), + ] = None, + config: Annotated[ + str | None, + Option( + "--config", + "-c", + help="Module configuration as JSON string.", + ), + ] = None, +) -> None: + """Execute a module. + + :param context: Typer context. + :param module_identifier: Module to execute. + :param assets_path: Optional path to input assets. + :param config: Optional JSON configuration. + + """ + import json + + runner = get_runner(context) + project_path = get_project_path(context) + + configuration: dict[str, Any] | None = None + if config: + try: + configuration = json.loads(config) + except json.JSONDecodeError as e: + console = Console() + console.print(f"[red]✗[/red] Invalid JSON configuration: {e}") + return + + console = Console() + console.print(f"[blue]→[/blue] Executing module: {module_identifier}") + + async def execute() -> None: + result = await runner.execute_module( + module_identifier=module_identifier, + project_path=project_path, + configuration=configuration, + assets_path=assets_path, + ) + + if result.success: + console.print(f"[green]✓[/green] Module execution completed") + console.print(f" Execution ID: {result.execution_id}") + console.print(f" Results: {result.results_path}") + else: + console.print(f"[red]✗[/red] Module execution failed") + console.print(f" Error: {result.error}") + + asyncio.run(execute()) + + +@application.command( + help="Show module information.", + name="info", +) +def module_info( + context: Context, + module_identifier: Annotated[ + str, + Argument( + help="Identifier of the module.", + ), + ], +) -> None: + """Show information about a specific module. + + :param context: Typer context. + :param module_identifier: Module to get info for. + + """ + runner = get_runner(context) + module = runner.get_module_info(module_identifier) + + console = Console() + + if module is None: + console.print(f"[red]✗[/red] Module not found: {module_identifier}") + return + + table = Table(title=f"Module: {module.identifier}") + table.add_column("Property", style="cyan") + table.add_column("Value") + + table.add_row("Identifier", module.identifier) + table.add_row("Available", "Yes" if module.available else "No") + table.add_row("Description", module.description or "-") + table.add_row("Version", module.version or "-") + + console.print(table) diff --git a/fuzzforge-cli/src/fuzzforge_cli/commands/projects.py b/fuzzforge-cli/src/fuzzforge_cli/commands/projects.py new file mode 100644 index 0000000..8be9f58 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/commands/projects.py @@ -0,0 +1,186 @@ +"""Project management commands for FuzzForge CLI.""" + +from pathlib import Path +from typing import Annotated + +from rich.console import Console +from rich.table import Table +from typer import Argument, Context, Option, Typer + +from fuzzforge_cli.context import get_project_path, get_runner + +application: Typer = Typer( + name="project", + help="Project management commands.", +) + + +@application.command( + help="Initialize a new FuzzForge project.", + name="init", +) +def init_project( + context: Context, + path: Annotated[ + Path | None, + Argument( + help="Path to initialize the project in. Defaults to current directory.", + ), + ] = None, +) -> None: + """Initialize a new FuzzForge project. + + Creates the necessary storage directories for the project. + + :param context: Typer context. + :param path: Path to initialize (defaults to current directory). + + """ + runner = get_runner(context) + project_path = path or get_project_path(context) + + storage_path = runner.init_project(project_path) + + console = Console() + console.print(f"[green]✓[/green] Project initialized at {project_path}") + console.print(f" Storage: {storage_path}") + + +@application.command( + help="Set project assets.", + name="assets", +) +def set_assets( + context: Context, + assets_path: Annotated[ + Path, + Argument( + help="Path to assets file or directory.", + ), + ], +) -> None: + """Set the initial assets for the project. + + :param context: Typer context. + :param assets_path: Path to assets. + + """ + runner = get_runner(context) + project_path = get_project_path(context) + + stored_path = runner.set_project_assets(project_path, assets_path) + + console = Console() + console.print(f"[green]✓[/green] Assets stored from {assets_path}") + console.print(f" Location: {stored_path}") + + +@application.command( + help="Show project information.", + name="info", +) +def show_info( + context: Context, +) -> None: + """Show information about the current project. + + :param context: Typer context. + + """ + runner = get_runner(context) + project_path = get_project_path(context) + + executions = runner.list_executions(project_path) + assets_path = runner.storage.get_project_assets_path(project_path) + + console = Console() + table = Table(title=f"Project: {project_path.name}") + table.add_column("Property", style="cyan") + table.add_column("Value") + + table.add_row("Path", str(project_path)) + table.add_row("Has Assets", "Yes" if assets_path else "No") + table.add_row("Assets Path", str(assets_path) if assets_path else "-") + table.add_row("Executions", str(len(executions))) + + console.print(table) + + +@application.command( + help="List all executions.", + name="executions", +) +def list_executions( + context: Context, +) -> None: + """List all executions for the project. + + :param context: Typer context. + + """ + runner = get_runner(context) + project_path = get_project_path(context) + + executions = runner.list_executions(project_path) + + console = Console() + + if not executions: + console.print("[yellow]No executions found.[/yellow]") + return + + table = Table(title="Executions") + table.add_column("ID", style="cyan") + table.add_column("Has Results") + + for exec_id in executions: + has_results = runner.get_execution_results(project_path, exec_id) is not None + table.add_row(exec_id, "✓" if has_results else "-") + + console.print(table) + + +@application.command( + help="Get execution results.", + name="results", +) +def get_results( + context: Context, + execution_id: Annotated[ + str, + Argument( + help="Execution ID to get results for.", + ), + ], + extract_to: Annotated[ + Path | None, + Option( + "--extract", + "-x", + help="Extract results to this directory.", + ), + ] = None, +) -> None: + """Get results for a specific execution. + + :param context: Typer context. + :param execution_id: Execution ID. + :param extract_to: Optional directory to extract to. + + """ + runner = get_runner(context) + project_path = get_project_path(context) + + results_path = runner.get_execution_results(project_path, execution_id) + + console = Console() + + if results_path is None: + console.print(f"[red]✗[/red] No results found for execution {execution_id}") + return + + console.print(f"[green]✓[/green] Results: {results_path}") + + if extract_to: + extracted = runner.extract_results(results_path, extract_to) + console.print(f" Extracted to: {extracted}") diff --git a/fuzzforge-cli/src/fuzzforge_cli/context.py b/fuzzforge-cli/src/fuzzforge_cli/context.py new file mode 100644 index 0000000..c53a061 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/context.py @@ -0,0 +1,64 @@ +"""FuzzForge CLI context management.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from fuzzforge_runner import Runner, Settings + +if TYPE_CHECKING: + from typer import Context as TyperContext + + +class Context: + """CLI context holding the runner instance and settings.""" + + _runner: Runner + _project_path: Path + + def __init__(self, runner: Runner, project_path: Path) -> None: + """Initialize an instance of the class. + + :param runner: FuzzForge runner instance. + :param project_path: Path to the current project. + + """ + self._runner = runner + self._project_path = project_path + + def get_runner(self) -> Runner: + """Get the runner instance. + + :return: Runner instance. + + """ + return self._runner + + def get_project_path(self) -> Path: + """Get the current project path. + + :return: Project path. + + """ + return self._project_path + + +def get_runner(context: TyperContext) -> Runner: + """Get runner from Typer context. + + :param context: Typer context. + :return: Runner instance. + + """ + return cast("Context", context.obj).get_runner() + + +def get_project_path(context: TyperContext) -> Path: + """Get project path from Typer context. + + :param context: Typer context. + :return: Project path. + + """ + return cast("Context", context.obj).get_project_path() diff --git a/fuzzforge-cli/src/fuzzforge_cli/utilities.py b/fuzzforge-cli/src/fuzzforge_cli/utilities.py new file mode 100644 index 0000000..1190455 --- /dev/null +++ b/fuzzforge-cli/src/fuzzforge_cli/utilities.py @@ -0,0 +1,18 @@ +"""CLI utility functions.""" + +from rich.console import Console +from rich.table import Table +from typer import Exit + + +def on_error(message: str) -> None: + """Display an error message and exit. + + :param message: Error message to display. + + """ + table = Table() + table.add_column("Error") + table.add_row(message) + Console().print(table) + raise Exit(code=1) diff --git a/fuzzforge-cli/tests/__init__.py b/fuzzforge-cli/tests/__init__.py new file mode 100644 index 0000000..4578657 --- /dev/null +++ b/fuzzforge-cli/tests/__init__.py @@ -0,0 +1 @@ +"""TODO.""" diff --git a/fuzzforge-cli/tests/conftest.py b/fuzzforge-cli/tests/conftest.py new file mode 100644 index 0000000..4578657 --- /dev/null +++ b/fuzzforge-cli/tests/conftest.py @@ -0,0 +1 @@ +"""TODO.""" diff --git a/fuzzforge-common/Makefile b/fuzzforge-common/Makefile new file mode 100644 index 0000000..0d7bf73 --- /dev/null +++ b/fuzzforge-common/Makefile @@ -0,0 +1,42 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +ARTIFACTS?=./dist + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit clean format mypy pytest ruff version wheel + +bandit: + uv run bandit --recursive $(SOURCES) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' + +wheel: + uv build --out-dir $(ARTIFACTS) diff --git a/fuzzforge-common/README.md b/fuzzforge-common/README.md new file mode 100644 index 0000000..f1819b8 --- /dev/null +++ b/fuzzforge-common/README.md @@ -0,0 +1,3 @@ +# FuzzForge Common + +... diff --git a/fuzzforge-common/mypy.ini b/fuzzforge-common/mypy.ini new file mode 100644 index 0000000..161c2a4 --- /dev/null +++ b/fuzzforge-common/mypy.ini @@ -0,0 +1,12 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True + +[mypy-botocore.*] +ignore_missing_imports = True + +[mypy-boto3.*] +ignore_missing_imports = True diff --git a/fuzzforge-common/pyproject.toml b/fuzzforge-common/pyproject.toml new file mode 100644 index 0000000..4a2f96c --- /dev/null +++ b/fuzzforge-common/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "fuzzforge-common" +version = "0.0.1" +description = "FuzzForge's common types and utilities." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-types==0.0.1", + "podman==5.6.0", + "pydantic==2.12.4", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[tool.uv.sources] +fuzzforge-types = { workspace = true } diff --git a/fuzzforge-common/pytest.ini b/fuzzforge-common/pytest.ini new file mode 100644 index 0000000..d4ecde6 --- /dev/null +++ b/fuzzforge-common/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +env = + DOCKER_HOST=unix:///run/user/1000/podman/podman.sock diff --git a/fuzzforge-common/ruff.toml b/fuzzforge-common/ruff.toml new file mode 100644 index 0000000..f8c919b --- /dev/null +++ b/fuzzforge-common/ruff.toml @@ -0,0 +1,20 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "ANN401", # allowing 'typing.Any' to be used to type function parameters in tests + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-common/src/fuzzforge_common/__init__.py b/fuzzforge-common/src/fuzzforge_common/__init__.py new file mode 100644 index 0000000..e2e81e4 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/__init__.py @@ -0,0 +1,54 @@ +"""FuzzForge Common - Shared abstractions and implementations for FuzzForge. + +This package provides: +- Sandbox engine abstractions (Podman, Docker) +- Storage abstractions (S3) - requires 'storage' extra +- Common exceptions + +Example usage: + from fuzzforge_common import ( + AbstractFuzzForgeSandboxEngine, + ImageInfo, + Podman, + PodmanConfiguration, + ) + + # For storage (requires boto3): + from fuzzforge_common.storage import Storage +""" + +from fuzzforge_common.exceptions import FuzzForgeError +from fuzzforge_common.sandboxes import ( + AbstractFuzzForgeEngineConfiguration, + AbstractFuzzForgeSandboxEngine, + Docker, + DockerConfiguration, + FuzzForgeSandboxEngines, + ImageInfo, + Podman, + PodmanConfiguration, +) + +# Storage exceptions are always available (no boto3 required) +from fuzzforge_common.storage.exceptions import ( + FuzzForgeStorageError, + StorageConnectionError, + StorageDownloadError, + StorageUploadError, +) + +__all__ = [ + "AbstractFuzzForgeEngineConfiguration", + "AbstractFuzzForgeSandboxEngine", + "Docker", + "DockerConfiguration", + "FuzzForgeError", + "FuzzForgeSandboxEngines", + "FuzzForgeStorageError", + "ImageInfo", + "Podman", + "PodmanConfiguration", + "StorageConnectionError", + "StorageDownloadError", + "StorageUploadError", +] diff --git a/fuzzforge-common/src/fuzzforge_common/exceptions.py b/fuzzforge-common/src/fuzzforge_common/exceptions.py new file mode 100644 index 0000000..c3fa4ae --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/exceptions.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + + +class FuzzForgeError(Exception): + """Base exception for all FuzzForge custom exceptions. + + All domain exceptions should inherit from this base to enable + consistent exception handling and hierarchy navigation. + + """ + + def __init__(self, message: str, details: dict[str, Any] | None = None) -> None: + """Initialize FuzzForge error. + + :param message: Error message. + :param details: Optional error details dictionary. + + """ + Exception.__init__(self, message) + self.message = message + self.details = details or {} diff --git a/fuzzforge-common/src/fuzzforge_common/py.typed b/fuzzforge-common/src/fuzzforge_common/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/__init__.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/__init__.py new file mode 100644 index 0000000..5efd40b --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/__init__.py @@ -0,0 +1,23 @@ +"""FuzzForge sandbox abstractions and implementations.""" + +from fuzzforge_common.sandboxes.engines import ( + AbstractFuzzForgeEngineConfiguration, + AbstractFuzzForgeSandboxEngine, + Docker, + DockerConfiguration, + FuzzForgeSandboxEngines, + ImageInfo, + Podman, + PodmanConfiguration, +) + +__all__ = [ + "AbstractFuzzForgeEngineConfiguration", + "AbstractFuzzForgeSandboxEngine", + "Docker", + "DockerConfiguration", + "FuzzForgeSandboxEngines", + "ImageInfo", + "Podman", + "PodmanConfiguration", +] diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/__init__.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/__init__.py new file mode 100644 index 0000000..f1a7c86 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/__init__.py @@ -0,0 +1,21 @@ +"""Container engine implementations for FuzzForge sandboxes.""" + +from fuzzforge_common.sandboxes.engines.base import ( + AbstractFuzzForgeEngineConfiguration, + AbstractFuzzForgeSandboxEngine, + ImageInfo, +) +from fuzzforge_common.sandboxes.engines.docker import Docker, DockerConfiguration +from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines +from fuzzforge_common.sandboxes.engines.podman import Podman, PodmanConfiguration + +__all__ = [ + "AbstractFuzzForgeEngineConfiguration", + "AbstractFuzzForgeSandboxEngine", + "Docker", + "DockerConfiguration", + "FuzzForgeSandboxEngines", + "ImageInfo", + "Podman", + "PodmanConfiguration", +] diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/__init__.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/__init__.py new file mode 100644 index 0000000..5ebf022 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/__init__.py @@ -0,0 +1,15 @@ +"""Base engine abstractions.""" + +from fuzzforge_common.sandboxes.engines.base.configuration import ( + AbstractFuzzForgeEngineConfiguration, +) +from fuzzforge_common.sandboxes.engines.base.engine import ( + AbstractFuzzForgeSandboxEngine, + ImageInfo, +) + +__all__ = [ + "AbstractFuzzForgeEngineConfiguration", + "AbstractFuzzForgeSandboxEngine", + "ImageInfo", +] diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/configuration.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/configuration.py new file mode 100644 index 0000000..aaae960 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/configuration.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from pydantic import BaseModel + +from fuzzforge_common.sandboxes.engines.enumeration import ( + FuzzForgeSandboxEngines, # noqa: TC001 (required by 'pydantic' at runtime) +) + +if TYPE_CHECKING: + from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine + + +class AbstractFuzzForgeEngineConfiguration(ABC, BaseModel): + """TODO.""" + + #: TODO. + kind: FuzzForgeSandboxEngines + + @abstractmethod + def into_engine(self) -> AbstractFuzzForgeSandboxEngine: + """TODO.""" + message: str = f"method 'into_engine' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/engine.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/engine.py new file mode 100644 index 0000000..aedca65 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/base/engine.py @@ -0,0 +1,281 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path, PurePath + + +@dataclass +class ImageInfo: + """Information about a container image.""" + + #: Full image reference (e.g., "localhost/fuzzforge-module-echidna:latest"). + reference: str + + #: Repository name (e.g., "localhost/fuzzforge-module-echidna"). + repository: str + + #: Image tag (e.g., "latest"). + tag: str + + #: Image ID (short hash). + image_id: str | None = None + + #: Image size in bytes. + size: int | None = None + + +class AbstractFuzzForgeSandboxEngine(ABC): + """Abstract class used as a base for all FuzzForge sandbox engine classes.""" + + @abstractmethod + def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]: + """List available container images. + + :param filter_prefix: Optional prefix to filter images (e.g., "localhost/"). + :returns: List of ImageInfo objects for available images. + + """ + message: str = f"method 'list_images' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def register_archive(self, archive: Path, repository: str) -> None: + """TODO. + + :param archive: TODO. + + """ + message: str = f"method 'register_archive' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def spawn_sandbox(self, image: str) -> str: + """Spawn a sandbox based on the given image. + + :param image: The image the sandbox should be based on. + :returns: The sandbox identifier. + + """ + message: str = f"method 'spawn_sandbox' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None: + """TODO. + + :param identifier: TODO. + :param source: TODO. + :param destination: TODO. + + """ + message: str = f"method 'push_archive_to_sandbox' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def start_sandbox(self, identifier: str) -> None: + """TODO. + + :param identifier: The identifier of the sandbox to start. + + """ + message: str = f"method 'start_sandbox' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None: + """Execute a command inside the sandbox matching the given identifier and wait for completion. + + :param sandbox: The identifier of the sandbox. + :param command: The command to run. + + """ + message: str = f"method 'execute_inside_sandbox' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path: + """TODO. + + :param identifier: TODO. + :param source: TODO. + :returns: TODO. + + """ + message: str = f"method 'pull_archive_from_sandbox' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def terminate_sandbox(self, identifier: str) -> None: + """Terminate the sandbox matching the given identifier. + + :param identifier: The identifier of the sandbox to terminate. + + """ + message: str = f"method 'terminate_sandbox' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + # ------------------------------------------------------------------------- + # Extended Container Operations + # ------------------------------------------------------------------------- + + @abstractmethod + def image_exists(self, image: str) -> bool: + """Check if a container image exists locally. + + :param image: Full image reference (e.g., "localhost/module:latest"). + :returns: True if image exists, False otherwise. + + """ + message: str = f"method 'image_exists' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def pull_image(self, image: str, timeout: int = 300) -> None: + """Pull an image from a container registry. + + :param image: Full image reference to pull. + :param timeout: Timeout in seconds for the pull operation. + :raises FuzzForgeError: If pull fails. + + """ + message: str = f"method 'pull_image' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def tag_image(self, source: str, target: str) -> None: + """Tag an image with a new name. + + :param source: Source image reference. + :param target: Target image reference. + + """ + message: str = f"method 'tag_image' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def create_container( + self, + image: str, + volumes: dict[str, str] | None = None, + ) -> str: + """Create a container from an image. + + :param image: Image to create container from. + :param volumes: Optional volume mappings {host_path: container_path}. + :returns: Container identifier. + + """ + message: str = f"method 'create_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def start_container_attached( + self, + identifier: str, + timeout: int = 600, + ) -> tuple[int, str, str]: + """Start a container and wait for it to complete. + + :param identifier: Container identifier. + :param timeout: Timeout in seconds for execution. + :returns: Tuple of (exit_code, stdout, stderr). + + """ + message: str = f"method 'start_container_attached' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def copy_to_container(self, identifier: str, source: Path, destination: str) -> None: + """Copy a file or directory to a container. + + :param identifier: Container identifier. + :param source: Source path on host. + :param destination: Destination path in container. + + """ + message: str = f"method 'copy_to_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def copy_from_container(self, identifier: str, source: str, destination: Path) -> None: + """Copy a file or directory from a container. + + :param identifier: Container identifier. + :param source: Source path in container. + :param destination: Destination path on host. + + """ + message: str = f"method 'copy_from_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def remove_container(self, identifier: str, *, force: bool = False) -> None: + """Remove a container. + + :param identifier: Container identifier. + :param force: Force removal even if running. + + """ + message: str = f"method 'remove_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + # ------------------------------------------------------------------------- + # Continuous/Background Execution Operations + # ------------------------------------------------------------------------- + + @abstractmethod + def start_container(self, identifier: str) -> None: + """Start a container without waiting for it to complete (detached mode). + + :param identifier: Container identifier. + + """ + message: str = f"method 'start_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def get_container_status(self, identifier: str) -> str: + """Get the status of a container. + + :param identifier: Container identifier. + :returns: Container status (e.g., "running", "exited", "created"). + + """ + message: str = f"method 'get_container_status' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def stop_container(self, identifier: str, timeout: int = 10) -> None: + """Stop a running container gracefully. + + :param identifier: Container identifier. + :param timeout: Seconds to wait before killing. + + """ + message: str = f"method 'stop_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def read_file_from_container(self, identifier: str, path: str) -> str: + """Read a file from inside a running container using exec. + + :param identifier: Container identifier. + :param path: Path to file inside container. + :returns: File contents as string. + + """ + message: str = f"method 'read_file_from_container' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def list_containers(self, all_containers: bool = True) -> list[dict]: + """List containers. + + :param all_containers: Include stopped containers. + :returns: List of container info dicts. + + """ + message: str = f"method 'list_containers' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/__init__.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/__init__.py new file mode 100644 index 0000000..3c46d08 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/__init__.py @@ -0,0 +1,11 @@ +"""Docker container engine implementation.""" + +from fuzzforge_common.sandboxes.engines.docker.configuration import ( + DockerConfiguration, +) +from fuzzforge_common.sandboxes.engines.docker.engine import Docker + +__all__ = [ + "Docker", + "DockerConfiguration", +] diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/configuration.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/configuration.py new file mode 100644 index 0000000..321e53f --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/configuration.py @@ -0,0 +1,22 @@ +from typing import TYPE_CHECKING, Literal + +from fuzzforge_common.sandboxes.engines.base.configuration import AbstractFuzzForgeEngineConfiguration +from fuzzforge_common.sandboxes.engines.docker.engine import Docker +from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines + +if TYPE_CHECKING: + from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine + + +class DockerConfiguration(AbstractFuzzForgeEngineConfiguration): + """TODO.""" + + #: TODO. + kind: Literal[FuzzForgeSandboxEngines.DOCKER] = FuzzForgeSandboxEngines.DOCKER + + #: TODO. + socket: str + + def into_engine(self) -> AbstractFuzzForgeSandboxEngine: + """TODO.""" + return Docker(socket=self.socket) diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/engine.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/engine.py new file mode 100644 index 0000000..18e2d99 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/docker/engine.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo + +if TYPE_CHECKING: + from pathlib import Path, PurePath + + +class Docker(AbstractFuzzForgeSandboxEngine): + """TODO.""" + + #: TODO. + __socket: str + + def __init__(self, socket: str) -> None: + """Initialize an instance of the class. + + :param socket: TODO. + + """ + super().__init__() + self.__socket = socket + + def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]: + """List available container images. + + :param filter_prefix: Optional prefix to filter images (e.g., "localhost/"). + :returns: List of ImageInfo objects for available images. + + """ + # TODO: Implement Docker image listing + message: str = "Docker engine list_images is not yet implemented" + raise NotImplementedError(message) + + def register_archive(self, archive: Path, repository: str) -> None: + """TODO. + + :param archive: TODO. + + """ + return super().register_archive(archive=archive, repository=repository) + + def spawn_sandbox(self, image: str) -> str: + """Spawn a sandbox based on the given image. + + :param image: The image the sandbox should be based on. + :returns: The sandbox identifier. + + """ + return super().spawn_sandbox(image) + + def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None: + """TODO. + + :param identifier: TODO. + :param source: TODO. + :param destination: TODO. + + """ + super().push_archive_to_sandbox(identifier, source, destination) + + def start_sandbox(self, identifier: str) -> None: + """TODO. + + :param identifier: The identifier of the sandbox to start. + + """ + super().start_sandbox(identifier) + + def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None: + """Execute a command inside the sandbox matching the given identifier and wait for completion. + + :param sandbox: The identifier of the sandbox. + :param command: The command to run. + + """ + super().execute_inside_sandbox(identifier, command) + + def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path: + """TODO. + + :param identifier: TODO. + :param source: TODO. + :returns: TODO. + + """ + return super().pull_archive_from_sandbox(identifier, source) + + def terminate_sandbox(self, identifier: str) -> None: + """Terminate the sandbox matching the given identifier. + + :param identifier: The identifier of the sandbox to terminate. + + """ + super().terminate_sandbox(identifier) + + # ------------------------------------------------------------------------- + # Extended Container Operations (stubs - not yet implemented) + # ------------------------------------------------------------------------- + + def image_exists(self, image: str) -> bool: + """Check if a container image exists locally.""" + message: str = "Docker engine image_exists is not yet implemented" + raise NotImplementedError(message) + + def pull_image(self, image: str, timeout: int = 300) -> None: + """Pull an image from a container registry.""" + message: str = "Docker engine pull_image is not yet implemented" + raise NotImplementedError(message) + + def tag_image(self, source: str, target: str) -> None: + """Tag an image with a new name.""" + message: str = "Docker engine tag_image is not yet implemented" + raise NotImplementedError(message) + + def create_container( + self, + image: str, + volumes: dict[str, str] | None = None, + ) -> str: + """Create a container from an image.""" + message: str = "Docker engine create_container is not yet implemented" + raise NotImplementedError(message) + + def start_container_attached( + self, + identifier: str, + timeout: int = 600, + ) -> tuple[int, str, str]: + """Start a container and wait for it to complete.""" + message: str = "Docker engine start_container_attached is not yet implemented" + raise NotImplementedError(message) + + def copy_to_container(self, identifier: str, source: Path, destination: str) -> None: + """Copy a file or directory to a container.""" + message: str = "Docker engine copy_to_container is not yet implemented" + raise NotImplementedError(message) + + def copy_from_container(self, identifier: str, source: str, destination: Path) -> None: + """Copy a file or directory from a container.""" + message: str = "Docker engine copy_from_container is not yet implemented" + raise NotImplementedError(message) + + def remove_container(self, identifier: str, *, force: bool = False) -> None: + """Remove a container.""" + message: str = "Docker engine remove_container is not yet implemented" + raise NotImplementedError(message) + + def start_container(self, identifier: str) -> None: + """Start a container without waiting for it to complete.""" + message: str = "Docker engine start_container is not yet implemented" + raise NotImplementedError(message) + + def get_container_status(self, identifier: str) -> str: + """Get the status of a container.""" + message: str = "Docker engine get_container_status is not yet implemented" + raise NotImplementedError(message) + + def stop_container(self, identifier: str, timeout: int = 10) -> None: + """Stop a running container gracefully.""" + message: str = "Docker engine stop_container is not yet implemented" + raise NotImplementedError(message) + + def read_file_from_container(self, identifier: str, path: str) -> str: + """Read a file from inside a running container using exec.""" + message: str = "Docker engine read_file_from_container is not yet implemented" + raise NotImplementedError(message) + + def list_containers(self, all_containers: bool = True) -> list[dict]: + """List containers.""" + message: str = "Docker engine list_containers is not yet implemented" + raise NotImplementedError(message) diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/enumeration.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/enumeration.py new file mode 100644 index 0000000..8de5650 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/enumeration.py @@ -0,0 +1,11 @@ +from enum import StrEnum + + +class FuzzForgeSandboxEngines(StrEnum): + """TODO.""" + + #: TODO. + DOCKER = "docker" + + #: TODO. + PODMAN = "podman" diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/__init__.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/__init__.py new file mode 100644 index 0000000..059af8b --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/__init__.py @@ -0,0 +1,13 @@ +"""Podman container engine implementation.""" + +from fuzzforge_common.sandboxes.engines.podman.cli import PodmanCLI +from fuzzforge_common.sandboxes.engines.podman.configuration import ( + PodmanConfiguration, +) +from fuzzforge_common.sandboxes.engines.podman.engine import Podman + +__all__ = [ + "Podman", + "PodmanCLI", + "PodmanConfiguration", +] diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/cli.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/cli.py new file mode 100644 index 0000000..24dfa2f --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/cli.py @@ -0,0 +1,444 @@ +"""Podman CLI engine with custom storage support. + +This engine uses subprocess calls to the Podman CLI instead of the socket API, +allowing for custom storage paths (--root, --runroot) that work regardless of +system Podman configuration or snap environment issues. +""" + +from __future__ import annotations + +import json +import subprocess +import tarfile +from io import BytesIO +from pathlib import Path, PurePath +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, cast + +from fuzzforge_common.exceptions import FuzzForgeError +from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo + +if TYPE_CHECKING: + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """Get structured logger.""" + from structlog import get_logger # noqa: PLC0415 (required by temporal) + + return cast("BoundLogger", get_logger()) + + +class PodmanCLI(AbstractFuzzForgeSandboxEngine): + """Podman engine using CLI with custom storage paths. + + This implementation uses subprocess calls to the Podman CLI with --root + and --runroot flags, providing isolation from system Podman storage. + This is particularly useful when running from VS Code snap which sets + XDG_DATA_HOME to a version-specific path. + """ + + __graphroot: Path + __runroot: Path + + def __init__(self, graphroot: Path, runroot: Path) -> None: + """Initialize the PodmanCLI engine. + + :param graphroot: Path to container image storage. + :param runroot: Path to container runtime state. + + """ + AbstractFuzzForgeSandboxEngine.__init__(self) + self.__graphroot = graphroot + self.__runroot = runroot + + # Ensure directories exist + self.__graphroot.mkdir(parents=True, exist_ok=True) + self.__runroot.mkdir(parents=True, exist_ok=True) + + def _base_cmd(self) -> list[str]: + """Get base Podman command with storage flags. + + :returns: Base command list with --root and --runroot. + + """ + return [ + "podman", + "--root", str(self.__graphroot), + "--runroot", str(self.__runroot), + ] + + def _run(self, args: list[str], *, check: bool = True, capture: bool = True) -> subprocess.CompletedProcess: + """Run a Podman command. + + :param args: Command arguments (without 'podman'). + :param check: Raise exception on non-zero exit. + :param capture: Capture stdout/stderr. + :returns: CompletedProcess result. + + """ + cmd = self._base_cmd() + args + get_logger().debug("running podman command", cmd=" ".join(cmd)) + return subprocess.run( + cmd, + check=check, + capture_output=capture, + text=True, + ) + + # ------------------------------------------------------------------------- + # Image Operations + # ------------------------------------------------------------------------- + + def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]: + """List available container images. + + :param filter_prefix: Optional prefix to filter images. + :returns: List of ImageInfo objects. + + """ + result = self._run(["images", "--format", "json"]) + images: list[ImageInfo] = [] + + try: + data = json.loads(result.stdout) if result.stdout.strip() else [] + except json.JSONDecodeError: + get_logger().warning("failed to parse podman images output") + return images + + for image in data: + # Get repository and tag from Names + names = image.get("Names") or [] + for name in names: + if filter_prefix and not name.startswith(filter_prefix): + continue + + # Parse repository and tag + if ":" in name: + repo, tag = name.rsplit(":", 1) + else: + repo = name + tag = "latest" + + images.append( + ImageInfo( + reference=name, + repository=repo, + tag=tag, + image_id=image.get("Id", "")[:12], + size=image.get("Size"), + ) + ) + + get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix) + return images + + def image_exists(self, image: str) -> bool: + """Check if a container image exists locally. + + :param image: Full image reference. + :returns: True if image exists. + + """ + result = self._run(["image", "exists", image], check=False) + return result.returncode == 0 + + def pull_image(self, image: str, timeout: int = 300) -> None: + """Pull an image from a container registry. + + :param image: Full image reference. + :param timeout: Timeout in seconds. + + """ + get_logger().info("pulling image", image=image) + try: + self._run(["pull", image]) + get_logger().info("image pulled successfully", image=image) + except subprocess.CalledProcessError as exc: + message = f"Failed to pull image '{image}': {exc.stderr}" + raise FuzzForgeError(message) from exc + + def tag_image(self, source: str, target: str) -> None: + """Tag an image with a new name. + + :param source: Source image reference. + :param target: Target image reference. + + """ + self._run(["tag", source, target]) + get_logger().debug("tagged image", source=source, target=target) + + def build_image(self, context_path: Path, tag: str, dockerfile: str = "Dockerfile") -> None: + """Build an image from a Dockerfile. + + :param context_path: Path to build context. + :param tag: Image tag. + :param dockerfile: Dockerfile name. + + """ + get_logger().info("building image", tag=tag, context=str(context_path)) + self._run(["build", "-t", tag, "-f", dockerfile, str(context_path)]) + get_logger().info("image built successfully", tag=tag) + + def register_archive(self, archive: Path, repository: str) -> None: + """Load an image from a tar archive. + + :param archive: Path to tar archive. + :param repository: Repository name for the loaded image. + + """ + result = self._run(["load", "-i", str(archive)]) + # Tag the loaded image + # Parse loaded image ID from output + for line in result.stdout.splitlines(): + if "Loaded image:" in line: + loaded_image = line.split("Loaded image:")[-1].strip() + self._run(["tag", loaded_image, f"{repository}:latest"]) + break + get_logger().debug("registered archive", archive=str(archive), repository=repository) + + # ------------------------------------------------------------------------- + # Container Operations + # ------------------------------------------------------------------------- + + def spawn_sandbox(self, image: str) -> str: + """Spawn a sandbox (container) from an image. + + :param image: Image to create container from. + :returns: Container identifier. + + """ + result = self._run(["create", image]) + container_id = result.stdout.strip() + get_logger().debug("created container", container_id=container_id) + return container_id + + def create_container( + self, + image: str, + volumes: dict[str, str] | None = None, + ) -> str: + """Create a container from an image. + + :param image: Image to create container from. + :param volumes: Optional volume mappings {host_path: container_path}. + :returns: Container identifier. + + """ + args = ["create"] + if volumes: + for host_path, container_path in volumes.items(): + args.extend(["-v", f"{host_path}:{container_path}:ro"]) + args.append(image) + + result = self._run(args) + container_id = result.stdout.strip() + get_logger().debug("created container", container_id=container_id, image=image) + return container_id + + def start_sandbox(self, identifier: str) -> None: + """Start a container. + + :param identifier: Container identifier. + + """ + self._run(["start", identifier]) + get_logger().debug("started container", container_id=identifier) + + def start_container(self, identifier: str) -> None: + """Start a container without waiting. + + :param identifier: Container identifier. + + """ + self._run(["start", identifier]) + get_logger().debug("started container (detached)", container_id=identifier) + + def start_container_attached( + self, + identifier: str, + timeout: int = 600, + ) -> tuple[int, str, str]: + """Start a container and wait for completion. + + :param identifier: Container identifier. + :param timeout: Timeout in seconds. + :returns: Tuple of (exit_code, stdout, stderr). + + """ + get_logger().debug("starting container attached", container_id=identifier) + # Start the container + self._run(["start", identifier]) + + # Wait for completion + wait_result = self._run(["wait", identifier]) + exit_code = int(wait_result.stdout.strip()) if wait_result.stdout.strip() else -1 + + # Get logs + stdout_result = self._run(["logs", identifier], check=False) + stdout_str = stdout_result.stdout or "" + stderr_str = stdout_result.stderr or "" + + get_logger().debug("container finished", container_id=identifier, exit_code=exit_code) + return (exit_code, stdout_str, stderr_str) + + def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None: + """Execute a command inside a container. + + :param identifier: Container identifier. + :param command: Command to run. + + """ + get_logger().debug("executing command in container", container_id=identifier) + self._run(["exec", identifier] + command) + + def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None: + """Copy an archive to a container. + + :param identifier: Container identifier. + :param source: Source archive path. + :param destination: Destination path in container. + + """ + get_logger().debug("copying to container", container_id=identifier, source=str(source)) + self._run(["cp", str(source), f"{identifier}:{destination}"]) + + def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path: + """Copy files from a container to a local archive. + + :param identifier: Container identifier. + :param source: Source path in container. + :returns: Path to local archive. + + """ + get_logger().debug("copying from container", container_id=identifier, source=str(source)) + with NamedTemporaryFile(delete=False, delete_on_close=False, suffix=".tar") as tmp: + self._run(["cp", f"{identifier}:{source}", tmp.name]) + return Path(tmp.name) + + def copy_to_container(self, identifier: str, source: Path, destination: str) -> None: + """Copy a file or directory to a container. + + :param identifier: Container identifier. + :param source: Source path on host. + :param destination: Destination path in container. + + """ + self._run(["cp", str(source), f"{identifier}:{destination}"]) + get_logger().debug("copied to container", source=str(source), destination=destination) + + def copy_from_container(self, identifier: str, source: str, destination: Path) -> None: + """Copy a file or directory from a container. + + :param identifier: Container identifier. + :param source: Source path in container. + :param destination: Destination path on host. + + """ + destination.mkdir(parents=True, exist_ok=True) + self._run(["cp", f"{identifier}:{source}", str(destination)]) + get_logger().debug("copied from container", source=source, destination=str(destination)) + + def terminate_sandbox(self, identifier: str) -> None: + """Terminate and remove a container. + + :param identifier: Container identifier. + + """ + # Stop if running + self._run(["stop", identifier], check=False) + # Remove + self._run(["rm", "-f", identifier], check=False) + get_logger().debug("terminated container", container_id=identifier) + + def remove_container(self, identifier: str, *, force: bool = False) -> None: + """Remove a container. + + :param identifier: Container identifier. + :param force: Force removal. + + """ + args = ["rm"] + if force: + args.append("-f") + args.append(identifier) + self._run(args, check=False) + get_logger().debug("removed container", container_id=identifier) + + def stop_container(self, identifier: str, timeout: int = 10) -> None: + """Stop a running container. + + :param identifier: Container identifier. + :param timeout: Seconds to wait before killing. + + """ + self._run(["stop", "-t", str(timeout), identifier], check=False) + get_logger().debug("stopped container", container_id=identifier) + + def get_container_status(self, identifier: str) -> str: + """Get the status of a container. + + :param identifier: Container identifier. + :returns: Container status. + + """ + result = self._run(["inspect", "--format", "{{.State.Status}}", identifier], check=False) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + def read_file_from_container(self, identifier: str, path: str) -> str: + """Read a file from inside a container. + + :param identifier: Container identifier. + :param path: Path to file in container. + :returns: File contents. + + """ + result = self._run(["exec", identifier, "cat", path], check=False) + if result.returncode != 0: + get_logger().debug("failed to read file from container", path=path) + return "" + return result.stdout + + def list_containers(self, all_containers: bool = True) -> list[dict]: + """List containers. + + :param all_containers: Include stopped containers. + :returns: List of container info dicts. + + """ + args = ["ps", "--format", "json"] + if all_containers: + args.append("-a") + + result = self._run(args) + try: + data = json.loads(result.stdout) if result.stdout.strip() else [] + # Handle both list and single object responses + if isinstance(data, dict): + data = [data] + return [ + { + "Id": c.get("Id", ""), + "Names": c.get("Names", []), + "Status": c.get("State", ""), + "Image": c.get("Image", ""), + } + for c in data + ] + except json.JSONDecodeError: + return [] + + # ------------------------------------------------------------------------- + # Utility Methods + # ------------------------------------------------------------------------- + + def get_storage_info(self) -> dict: + """Get storage configuration info. + + :returns: Dict with graphroot and runroot paths. + + """ + return { + "graphroot": str(self.__graphroot), + "runroot": str(self.__runroot), + } diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/configuration.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/configuration.py new file mode 100644 index 0000000..2669303 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/configuration.py @@ -0,0 +1,22 @@ +from typing import TYPE_CHECKING, Literal + +from fuzzforge_common.sandboxes.engines.base.configuration import AbstractFuzzForgeEngineConfiguration +from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines +from fuzzforge_common.sandboxes.engines.podman.engine import Podman + +if TYPE_CHECKING: + from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine + + +class PodmanConfiguration(AbstractFuzzForgeEngineConfiguration): + """TODO.""" + + #: TODO. + kind: Literal[FuzzForgeSandboxEngines.PODMAN] = FuzzForgeSandboxEngines.PODMAN + + #: TODO. + socket: str + + def into_engine(self) -> AbstractFuzzForgeSandboxEngine: + """TODO.""" + return Podman(socket=self.socket) diff --git a/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/engine.py b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/engine.py new file mode 100644 index 0000000..343365f --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/sandboxes/engines/podman/engine.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import tarfile +from io import BytesIO +from pathlib import Path, PurePath +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, cast + +from podman.errors import ImageNotFound + +from fuzzforge_common.exceptions import FuzzForgeError +from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo + +if TYPE_CHECKING: + from podman import PodmanClient + from podman.domain.containers import Container + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """TODO.""" + from structlog import get_logger # noqa: PLC0415 (required by temporal) + + return cast("BoundLogger", get_logger()) + + +class Podman(AbstractFuzzForgeSandboxEngine): + """TODO.""" + + #: TODO. + __socket: str + + def __init__(self, socket: str) -> None: + """Initialize an instance of the class. + + :param socket: TODO. + + """ + AbstractFuzzForgeSandboxEngine.__init__(self) + self.__socket = socket + + def get_client(self) -> PodmanClient: + """TODO. + + :returns TODO. + + """ + from podman import PodmanClient # noqa: PLC0415 (required by temporal) + + return PodmanClient(base_url=self.__socket) + + def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]: + """List available container images. + + :param filter_prefix: Optional prefix to filter images (e.g., "localhost/"). + :returns: List of ImageInfo objects for available images. + + """ + client: PodmanClient = self.get_client() + images: list[ImageInfo] = [] + + with client: + for image in client.images.list(): + # Get all tags for this image + tags = image.tags or [] + for tag in tags: + # Apply filter if specified + if filter_prefix and not tag.startswith(filter_prefix): + continue + + # Parse repository and tag + if ":" in tag: + repo, tag_name = tag.rsplit(":", 1) + else: + repo = tag + tag_name = "latest" + + images.append( + ImageInfo( + reference=tag, + repository=repo, + tag=tag_name, + image_id=image.short_id if hasattr(image, "short_id") else image.id[:12], + size=image.attrs.get("Size") if hasattr(image, "attrs") else None, + ) + ) + + get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix) + return images + + def register_archive(self, archive: Path, repository: str) -> None: + """TODO. + + :param archive: TODO. + + """ + client: PodmanClient = self.get_client() + with client: + images = list(client.images.load(file_path=archive)) + if len(images) != 1: + message: str = "expected only one image" + raise FuzzForgeError(message) + image = images[0] + image.tag(repository=repository, tag="latest") + + def spawn_sandbox(self, image: str) -> str: + """Spawn a sandbox based on the given image. + + :param image: The image the sandbox should be based on. + :returns: The sandbox identifier. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.create(image=image) + container_identifier: str = container.id + get_logger().debug("create podman container", container_identifier=container_identifier) + return container_identifier + + def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None: + """TODO. + + :param identifier: TODO. + :param source: TODO. + :param destination: TODO. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + get_logger().debug( + "push archive to podman container", + container_identifier=identifier, + container_status=container.status, + ) + # reading everything at once for now, even though this temporary solution is not viable with large files, + # since the podman sdk does not currently expose a way to chunk uploads. + # in order to fix this issue, we could directly interact with the podman rest api or make a contribution + # to the podman sdk in order to allow the 'put_archive' method to support chunked uploads. + data: bytes = source.read_bytes() + container.put_archive(path=str(destination), data=data) + + def start_sandbox(self, identifier: str) -> None: + """Start the sandbox matching the given identifier. + + :param identifier: The identifier of the sandbox to start. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + get_logger().debug( + "start podman container", + container_identifier=identifier, + container_status=container.status, + ) + container.start() + + def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None: + """Execute a command inside the sandbox matching the given identifier and wait for completion. + + :param sandbox: The identifier of the sandbox. + :param command: The command to run. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + get_logger().debug( + "executing command inside podman container", + container_identifier=identifier, + container_status=container.status, + ) + (status, (stdout, stderr)) = container.exec_run(cmd=command, demux=True) + get_logger().debug( + "command execution result", + status=status, + stdout_size=len(stdout) if stdout else 0, + stderr_size=len(stderr) if stderr else 0, + ) + + def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path: + """TODO. + + :param identifier: TODO. + :param source: TODO. + :returns: TODO. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + get_logger().debug( + "pull archive from podman container", + container_identifier=identifier, + container_status=container.status, + ) + with NamedTemporaryFile(delete=False, delete_on_close=False) as file: + stream, _stat = container.get_archive(path=str(source)) + for chunk in stream: + file.write(chunk) + get_logger().debug( + "created archive", + archive=file.name, + ) + return Path(file.name) + + def terminate_sandbox(self, identifier: str) -> None: + """Terminate the sandbox matching the given identifier. + + :param identifier: The identifier of the sandbox to terminate. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + get_logger().debug( + "kill podman container", + container_identifier=identifier, + container_status=container.status, + ) + # Only kill running containers; for created/stopped, skip to remove + if container.status in ("running", "paused"): + container.kill() + get_logger().debug( + "remove podman container", + container_identifier=identifier, + container_status=container.status, + ) + container.remove() + + # ------------------------------------------------------------------------- + # Extended Container Operations + # ------------------------------------------------------------------------- + + def image_exists(self, image: str) -> bool: + """Check if a container image exists locally. + + :param image: Full image reference (e.g., "localhost/module:latest"). + :returns: True if image exists, False otherwise. + + """ + client: PodmanClient = self.get_client() + with client: + try: + client.images.get(name=image) + except ImageNotFound: + return False + else: + return True + + def pull_image(self, image: str, timeout: int = 300) -> None: + """Pull an image from a container registry. + + :param image: Full image reference to pull. + :param timeout: Timeout in seconds for the pull operation. + :raises FuzzForgeError: If pull fails. + + """ + client: PodmanClient = self.get_client() + with client: + try: + get_logger().info("pulling image", image=image) + client.images.pull(repository=image) + get_logger().info("image pulled successfully", image=image) + except Exception as exc: + message = f"Failed to pull image '{image}': {exc}" + raise FuzzForgeError(message) from exc + + def tag_image(self, source: str, target: str) -> None: + """Tag an image with a new name. + + :param source: Source image reference. + :param target: Target image reference. + + """ + client: PodmanClient = self.get_client() + with client: + image = client.images.get(name=source) + # Parse target into repository and tag + if ":" in target: + repo, tag = target.rsplit(":", 1) + else: + repo = target + tag = "latest" + image.tag(repository=repo, tag=tag) + get_logger().debug("tagged image", source=source, target=target) + + def create_container( + self, + image: str, + volumes: dict[str, str] | None = None, + ) -> str: + """Create a container from an image. + + :param image: Image to create container from. + :param volumes: Optional volume mappings {host_path: container_path}. + :returns: Container identifier. + + """ + client: PodmanClient = self.get_client() + with client: + # Build volume mounts in podman format + mounts = [] + if volumes: + for host_path, container_path in volumes.items(): + mounts.append({"type": "bind", "source": host_path, "target": container_path, "read_only": True}) + + container: Container = client.containers.create(image=image, mounts=mounts if mounts else None) + container_id: str = str(container.id) + get_logger().debug("created container", container_id=container_id, image=image) + return container_id + + def start_container_attached( + self, + identifier: str, + timeout: int = 600, + ) -> tuple[int, str, str]: + """Start a container and wait for it to complete. + + :param identifier: Container identifier. + :param timeout: Timeout in seconds for execution. + :returns: Tuple of (exit_code, stdout, stderr). + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + get_logger().debug("starting container attached", container_id=identifier) + + # Start the container + container.start() + + # Wait for completion with timeout + result = container.wait(timeout=timeout) + exit_code: int = result.get("StatusCode", -1) if isinstance(result, dict) else int(result) + + # Get logs + stdout_raw = container.logs(stdout=True, stderr=False) + stderr_raw = container.logs(stdout=False, stderr=True) + + # Decode if bytes + stdout_str: str = "" + stderr_str: str = "" + if isinstance(stdout_raw, bytes): + stdout_str = stdout_raw.decode("utf-8", errors="replace") + elif isinstance(stdout_raw, str): + stdout_str = stdout_raw + if isinstance(stderr_raw, bytes): + stderr_str = stderr_raw.decode("utf-8", errors="replace") + elif isinstance(stderr_raw, str): + stderr_str = stderr_raw + + get_logger().debug("container finished", container_id=identifier, exit_code=exit_code) + return (exit_code, stdout_str, stderr_str) + + def copy_to_container(self, identifier: str, source: Path, destination: str) -> None: + """Copy a file or directory to a container. + + :param identifier: Container identifier. + :param source: Source path on host. + :param destination: Destination path in container. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + + # Create tar archive in memory + tar_buffer = BytesIO() + with tarfile.open(fileobj=tar_buffer, mode="w") as tar: + tar.add(str(source), arcname=Path(source).name) + tar_buffer.seek(0) + + # Use put_archive to copy + container.put_archive(path=destination, data=tar_buffer.read()) + get_logger().debug("copied to container", source=str(source), destination=destination) + + def copy_from_container(self, identifier: str, source: str, destination: Path) -> None: + """Copy a file or directory from a container. + + :param identifier: Container identifier. + :param source: Source path in container. + :param destination: Destination path on host. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + + # Get archive from container + stream, _stat = container.get_archive(path=source) + + # Write to temp file and extract + tar_buffer = BytesIO() + for chunk in stream: + tar_buffer.write(chunk) + tar_buffer.seek(0) + + # Extract to destination + destination.mkdir(parents=True, exist_ok=True) + with tarfile.open(fileobj=tar_buffer, mode="r") as tar: + tar.extractall(path=destination) # noqa: S202 (trusted source) + + get_logger().debug("copied from container", source=source, destination=str(destination)) + + def remove_container(self, identifier: str, *, force: bool = False) -> None: + """Remove a container. + + :param identifier: Container identifier. + :param force: Force removal even if running. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + if force and container.status in ("running", "paused"): + container.kill() + container.remove() + get_logger().debug("removed container", container_id=identifier) + + def start_container(self, identifier: str) -> None: + """Start a container without waiting for it to complete. + + :param identifier: Container identifier. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + container.start() + get_logger().debug("started container (detached)", container_id=identifier) + + def get_container_status(self, identifier: str) -> str: + """Get the status of a container. + + :param identifier: Container identifier. + :returns: Container status (e.g., "running", "exited", "created"). + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + return str(container.status) + + def stop_container(self, identifier: str, timeout: int = 10) -> None: + """Stop a running container gracefully. + + :param identifier: Container identifier. + :param timeout: Seconds to wait before killing. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + if container.status == "running": + container.stop(timeout=timeout) + get_logger().debug("stopped container", container_id=identifier) + + def read_file_from_container(self, identifier: str, path: str) -> str: + """Read a file from inside a running container using exec. + + :param identifier: Container identifier. + :param path: Path to file inside container. + :returns: File contents as string. + + """ + client: PodmanClient = self.get_client() + with client: + container: Container = client.containers.get(key=identifier) + (status, (stdout, stderr)) = container.exec_run(cmd=["cat", path], demux=True) + if status != 0: + error_msg = stderr.decode("utf-8", errors="replace") if stderr else "File not found" + get_logger().debug("failed to read file from container", path=path, error=error_msg) + return "" + return stdout.decode("utf-8", errors="replace") if stdout else "" + + def list_containers(self, all_containers: bool = True) -> list[dict]: + """List containers. + + :param all_containers: Include stopped containers. + :returns: List of container info dicts. + + """ + client: PodmanClient = self.get_client() + with client: + containers = client.containers.list(all=all_containers) + return [ + { + "Id": str(c.id), + "Names": [c.name] if hasattr(c, "name") else [], + "Status": str(c.status), + "Image": str(c.image) if hasattr(c, "image") else "", + } + for c in containers + ] diff --git a/fuzzforge-common/src/fuzzforge_common/storage/__init__.py b/fuzzforge-common/src/fuzzforge_common/storage/__init__.py new file mode 100644 index 0000000..77dcb2b --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/storage/__init__.py @@ -0,0 +1,19 @@ +"""FuzzForge storage abstractions. + +Storage class requires boto3. Import it explicitly: + from fuzzforge_common.storage.s3 import Storage +""" + +from fuzzforge_common.storage.exceptions import ( + FuzzForgeStorageError, + StorageConnectionError, + StorageDownloadError, + StorageUploadError, +) + +__all__ = [ + "FuzzForgeStorageError", + "StorageConnectionError", + "StorageDownloadError", + "StorageUploadError", +] diff --git a/fuzzforge-common/src/fuzzforge_common/storage/configuration.py b/fuzzforge-common/src/fuzzforge_common/storage/configuration.py new file mode 100644 index 0000000..1c56348 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/storage/configuration.py @@ -0,0 +1,20 @@ +from pydantic import BaseModel + +from fuzzforge_common.storage.s3 import Storage + + +class StorageConfiguration(BaseModel): + """TODO.""" + + #: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO). + endpoint: str + + #: S3 access key ID for authentication. + access_key: str + + #: S3 secret access key for authentication. + secret_key: str + + def into_storage(self) -> Storage: + """TODO.""" + return Storage(endpoint=self.endpoint, access_key=self.access_key, secret_key=self.secret_key) diff --git a/fuzzforge-common/src/fuzzforge_common/storage/exceptions.py b/fuzzforge-common/src/fuzzforge_common/storage/exceptions.py new file mode 100644 index 0000000..0c082e9 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/storage/exceptions.py @@ -0,0 +1,108 @@ +from fuzzforge_common.exceptions import FuzzForgeError + + +class FuzzForgeStorageError(FuzzForgeError): + """Base exception for all storage-related errors. + + Raised when storage operations (upload, download, connection) fail + during workflow execution. + + """ + + +class StorageConnectionError(FuzzForgeStorageError): + """Failed to connect to storage service. + + :param endpoint: The storage endpoint that failed to connect. + :param reason: The underlying exception message. + + """ + + def __init__(self, endpoint: str, reason: str) -> None: + """Initialize storage connection error. + + :param endpoint: The storage endpoint that failed to connect. + :param reason: The underlying exception message. + + """ + FuzzForgeStorageError.__init__( + self, + f"Failed to connect to storage at {endpoint}: {reason}", + ) + self.endpoint = endpoint + self.reason = reason + + +class StorageUploadError(FuzzForgeStorageError): + """Failed to upload object to storage. + + :param bucket: The target bucket name. + :param object_key: The target object key. + :param reason: The underlying exception message. + + """ + + def __init__(self, bucket: str, object_key: str, reason: str) -> None: + """Initialize storage upload error. + + :param bucket: The target bucket name. + :param object_key: The target object key. + :param reason: The underlying exception message. + + """ + FuzzForgeStorageError.__init__( + self, + f"Failed to upload to {bucket}/{object_key}: {reason}", + ) + self.bucket = bucket + self.object_key = object_key + self.reason = reason + + +class StorageDownloadError(FuzzForgeStorageError): + """Failed to download object from storage. + + :param bucket: The source bucket name. + :param object_key: The source object key. + :param reason: The underlying exception message. + + """ + + def __init__(self, bucket: str, object_key: str, reason: str) -> None: + """Initialize storage download error. + + :param bucket: The source bucket name. + :param object_key: The source object key. + :param reason: The underlying exception message. + + """ + FuzzForgeStorageError.__init__( + self, + f"Failed to download from {bucket}/{object_key}: {reason}", + ) + self.bucket = bucket + self.object_key = object_key + self.reason = reason + + +class StorageDeletionError(FuzzForgeStorageError): + """Failed to delete bucket from storage. + + :param bucket: The bucket name that failed to delete. + :param reason: The underlying exception message. + + """ + + def __init__(self, bucket: str, reason: str) -> None: + """Initialize storage deletion error. + + :param bucket: The bucket name that failed to delete. + :param reason: The underlying exception message. + + """ + FuzzForgeStorageError.__init__( + self, + f"Failed to delete bucket {bucket}: {reason}", + ) + self.bucket = bucket + self.reason = reason diff --git a/fuzzforge-common/src/fuzzforge_common/storage/s3.py b/fuzzforge-common/src/fuzzforge_common/storage/s3.py new file mode 100644 index 0000000..888cd84 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/storage/s3.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +from pathlib import Path, PurePath +from tarfile import TarInfo +from tarfile import open as Archive # noqa: N812 +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING, Any, cast + +from botocore.exceptions import ClientError + +from fuzzforge_common.storage.exceptions import StorageDeletionError, StorageDownloadError, StorageUploadError + +if TYPE_CHECKING: + from botocore.client import BaseClient + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """Get structlog logger instance. + + Uses deferred import pattern required by Temporal for serialization. + + :returns: Configured structlog logger. + + """ + from structlog import get_logger # noqa: PLC0415 (required by temporal) + + return cast("BoundLogger", get_logger()) + + +class Storage: + """S3-compatible storage backend implementation using boto3. + + Supports MinIO, AWS S3, and other S3-compatible storage services. + Uses error-driven approach (EAFP) to handle bucket creation and + avoid race conditions. + + """ + + #: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO). + __endpoint: str + + #: S3 access key ID for authentication. + __access_key: str + + #: S3 secret access key for authentication. + __secret_key: str + + def __init__(self, endpoint: str, access_key: str, secret_key: str) -> None: + """Initialize an instance of the class. + + :param endpoint: TODO. + :param access_key: TODO. + :param secret_key: TODO. + + """ + self.__endpoint = endpoint + self.__access_key = access_key + self.__secret_key = secret_key + + def _get_client(self) -> BaseClient: + """Create boto3 S3 client with configured credentials. + + Uses deferred import pattern required by Temporal for serialization. + + :returns: Configured boto3 S3 client. + + """ + import boto3 # noqa: PLC0415 (required by temporal) + + return boto3.client( + "s3", + endpoint_url=self.__endpoint, + aws_access_key_id=self.__access_key, + aws_secret_access_key=self.__secret_key, + ) + + def create_bucket(self, bucket: str) -> None: + """Create the S3 bucket if it does not already exist. + + Idempotent operation - succeeds if bucket already exists and is owned by you. + Fails if bucket exists but is owned by another account. + + :raise ClientError: If bucket creation fails (permissions, name conflicts, etc.). + + """ + logger = get_logger() + client = self._get_client() + + logger.debug("creating_bucket", bucket=bucket) + + try: + client.create_bucket(Bucket=bucket) + logger.info("bucket_created", bucket=bucket) + + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code") + + # Bucket already exists and we own it - this is fine + if error_code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + logger.debug( + "bucket_already_exists", + bucket=bucket, + error_code=error_code, + ) + return + + # Other errors are actual failures + logger.exception( + "bucket_creation_failed", + bucket=bucket, + error_code=error_code, + ) + raise + + def delete_bucket(self, bucket: str) -> None: + """Delete an S3 bucket and all its contents. + + Idempotent operation - succeeds if bucket doesn't exist. + Handles pagination for buckets with many objects. + + :param bucket: The name of the bucket to delete. + :raises StorageDeletionError: If bucket deletion fails. + + """ + logger = get_logger() + client = self._get_client() + + logger.debug("deleting_bucket", bucket=bucket) + + try: + # S3 requires bucket to be empty before deletion + # Delete all objects first with pagination support + continuation_token = None + + while True: + # List objects (up to 1000 per request) + list_params = {"Bucket": bucket} + if continuation_token: + list_params["ContinuationToken"] = continuation_token + + response = client.list_objects_v2(**list_params) + + # Delete objects if any exist (max 1000 per delete_objects call) + if "Contents" in response: + objects = [{"Key": obj["Key"]} for obj in response["Contents"]] + client.delete_objects(Bucket=bucket, Delete={"Objects": objects}) + logger.debug("deleted_objects", bucket=bucket, count=len(objects)) + + # Check if more objects exist + if not response.get("IsTruncated", False): + break + + continuation_token = response.get("NextContinuationToken") + + # Now delete the empty bucket + client.delete_bucket(Bucket=bucket) + logger.info("bucket_deleted", bucket=bucket) + + except ClientError as error: + error_code = error.response.get("Error", {}).get("Code") + + # Idempotent - bucket already doesn't exist + if error_code == "NoSuchBucket": + logger.debug("bucket_does_not_exist", bucket=bucket) + return + + # Other errors are actual failures + logger.exception( + "bucket_deletion_failed", + bucket=bucket, + error_code=error_code, + ) + raise StorageDeletionError(bucket=bucket, reason=str(error)) from error + + def upload_file( + self, + bucket: str, + file: Path, + key: str, + ) -> None: + """Upload archive file to S3 storage at specified object key. + + Assumes bucket exists. Fails gracefully if bucket or other resources missing. + + :param bucket: TODO. + :param file: Local path to the archive file to upload. + :param key: Object key (path) in S3 where file should be uploaded. + :raise StorageUploadError: If upload operation fails. + + """ + from boto3.exceptions import S3UploadFailedError # noqa: PLC0415 (required by 'temporal' at runtime) + + logger = get_logger() + client = self._get_client() + + logger.debug( + "uploading_archive_to_storage", + bucket=bucket, + object_key=key, + archive_path=str(file), + ) + + try: + client.upload_file( + Filename=str(file), + Bucket=bucket, + Key=key, + ) + logger.info( + "archive_uploaded_successfully", + bucket=bucket, + object_key=key, + ) + + except S3UploadFailedError as e: + # Check if this is a NoSuchBucket error - create bucket and retry + if "NoSuchBucket" in str(e): + logger.info( + "bucket_does_not_exist_creating", + bucket=bucket, + ) + self.create_bucket(bucket=bucket) + # Retry upload after creating bucket + try: + client.upload_file( + Filename=str(file), + Bucket=bucket, + Key=key, + ) + logger.info( + "archive_uploaded_successfully_after_bucket_creation", + bucket=bucket, + object_key=key, + ) + except S3UploadFailedError as retry_error: + logger.exception( + "upload_failed_after_bucket_creation", + bucket=bucket, + object_key=key, + ) + raise StorageUploadError( + bucket=bucket, + object_key=key, + reason=str(retry_error), + ) from retry_error + else: + logger.exception( + "upload_failed", + bucket=bucket, + object_key=key, + ) + raise StorageUploadError( + bucket=bucket, + object_key=key, + reason=str(e), + ) from e + + def download_file(self, bucket: str, key: PurePath) -> Path: + """Download a single file from S3 storage. + + Downloads the file to a temporary location and returns the path. + + :param bucket: S3 bucket name. + :param key: Object key (path) in S3 to download. + :returns: Path to the downloaded file. + :raise StorageDownloadError: If download operation fails. + + """ + logger = get_logger() + client = self._get_client() + + logger.debug( + "downloading_file_from_storage", + bucket=bucket, + object_key=str(key), + ) + + try: + # Create temporary file for download + with NamedTemporaryFile(delete=False, suffix=".tar.gz") as temp_file: + temp_path = Path(temp_file.name) + + # Download object to temp file + client.download_file( + Bucket=bucket, + Key=str(key), + Filename=str(temp_path), + ) + + logger.info( + "file_downloaded_successfully", + bucket=bucket, + object_key=str(key), + local_path=str(temp_path), + ) + + return temp_path + + except ClientError as error: + error_code = error.response.get("Error", {}).get("Code") + logger.exception( + "download_failed", + bucket=bucket, + object_key=str(key), + error_code=error_code, + ) + raise StorageDownloadError( + bucket=bucket, + object_key=str(key), + reason=f"{error_code}: {error!s}", + ) from error + + def download_directory(self, bucket: str, directory: PurePath) -> Path: + """TODO. + + :param bucket: TODO. + :param directory: TODO. + :returns: TODO. + + """ + with NamedTemporaryFile(delete=False) as file: + path: Path = Path(file.name) + # end-with + client: Any = self._get_client() + with Archive(name=str(path), mode="w:gz") as archive: + paginator = client.get_paginator("list_objects_v2") + try: + pages = paginator.paginate(Bucket=bucket, Prefix=str(directory)) + except ClientError as exception: + raise StorageDownloadError( + bucket=bucket, + object_key=str(directory), + reason=exception.response["Error"]["Code"], + ) from exception + for page in pages: + for entry in page.get("Contents", []): + key: str = entry["Key"] + try: + response: dict[str, Any] = client.get_object(Bucket=bucket, Key=key) + except ClientError as exception: + raise StorageDownloadError( + bucket=bucket, + object_key=key, + reason=exception.response["Error"]["Code"], + ) from exception + archive.addfile(TarInfo(name=key), fileobj=response["Body"]) + # end-for + # end-for + # end-with + return path diff --git a/fuzzforge-common/src/fuzzforge_common/temporal/__init__.py b/fuzzforge-common/src/fuzzforge_common/temporal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/src/fuzzforge_common/temporal/queues.py b/fuzzforge-common/src/fuzzforge_common/temporal/queues.py new file mode 100644 index 0000000..44b0dc3 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/temporal/queues.py @@ -0,0 +1,8 @@ +from enum import StrEnum + + +class TemporalQueues(StrEnum): + """Enumeration of available `Temporal Task Queues`.""" + + #: The default task queue. + DEFAULT = "default-task-queue" diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/__init__.py b/fuzzforge-common/src/fuzzforge_common/workflows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/base/__init__.py b/fuzzforge-common/src/fuzzforge_common/workflows/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/base/definitions.py b/fuzzforge-common/src/fuzzforge_common/workflows/base/definitions.py new file mode 100644 index 0000000..aed1615 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/workflows/base/definitions.py @@ -0,0 +1,46 @@ +from enum import StrEnum +from typing import Literal + +from fuzzforge_types import FuzzForgeWorkflowIdentifier # noqa: TC002 (required by 'pydantic' at runtime) +from pydantic import BaseModel + + +class Base(BaseModel): + """TODO.""" + + +class FuzzForgeWorkflowSteps(StrEnum): + """Workflow step types.""" + + #: Execute a FuzzForge module + RUN_FUZZFORGE_MODULE = "run-fuzzforge-module" + + +class FuzzForgeWorkflowStep(Base): + """TODO.""" + + #: The type of the workflow's step. + kind: FuzzForgeWorkflowSteps + + +class RunFuzzForgeModule(FuzzForgeWorkflowStep): + """Execute a FuzzForge module.""" + + kind: Literal[FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE] = FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE + #: The name of the module. + module: str + #: The container of the module. + container: str + + +class FuzzForgeWorkflowDefinition(Base): + """The definition of a FuzzForge workflow.""" + + #: The author of the workflow. + author: str + #: The identifier of the workflow. + identifier: FuzzForgeWorkflowIdentifier + #: The name of the workflow. + name: str + #: The collection of steps that compose the workflow. + steps: list[RunFuzzForgeModule] diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/base/parameters.py b/fuzzforge-common/src/fuzzforge_common/workflows/base/parameters.py new file mode 100644 index 0000000..c45df32 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/workflows/base/parameters.py @@ -0,0 +1,24 @@ +from pydantic import BaseModel + +from fuzzforge_common.sandboxes.engines.docker.configuration import ( + DockerConfiguration, # noqa: TC001 (required by pydantic at runtime) +) +from fuzzforge_common.sandboxes.engines.podman.configuration import ( + PodmanConfiguration, # noqa: TC001 (required by pydantic at runtime) +) +from fuzzforge_common.storage.configuration import StorageConfiguration # noqa: TC001 (required by pydantic at runtime) + + +class TemporalWorkflowParameters(BaseModel): + """Base parameters for Temporal workflows. + + Provides common configuration shared across all workflow types, + including sandbox engine and storage backend instances. + + """ + + #: Sandbox engine for container operations (Docker or Podman). + engine_configuration: PodmanConfiguration | DockerConfiguration + + #: Storage backend for uploading/downloading execution artifacts. + storage_configuration: StorageConfiguration diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/bridge_utils.py b/fuzzforge-common/src/fuzzforge_common/workflows/bridge_utils.py new file mode 100644 index 0000000..958f270 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/workflows/bridge_utils.py @@ -0,0 +1,108 @@ +"""Helper utilities for working with bridge transformations.""" + +from pathlib import Path +from typing import Any + + +def load_transform_from_file(file_path: str | Path) -> str: + """Load bridge transformation code from a Python file. + + This reads the transformation function from a .py file and extracts + the code as a string suitable for the bridge module. + + Args: + file_path: Path to Python file containing transform() function + + Returns: + Python code as a string + + Example: + >>> code = load_transform_from_file("transformations/add_line_numbers.py") + >>> # code contains the transform() function as a string + + """ + path = Path(file_path) + + if not path.exists(): + raise FileNotFoundError(f"Transformation file not found: {file_path}") + + if path.suffix != ".py": + raise ValueError(f"Transformation file must be .py file, got: {path.suffix}") + + # Read the entire file + code = path.read_text() + + return code + + +def create_bridge_input( + transform_file: str | Path, + input_filename: str | None = None, + output_filename: str | None = None, +) -> dict[str, Any]: + """Create bridge module input configuration from a transformation file. + + Args: + transform_file: Path to Python file with transform() function + input_filename: Optional specific input file to transform + output_filename: Optional specific output filename + + Returns: + Dictionary suitable for bridge module's input.json + + Example: + >>> config = create_bridge_input("transformations/add_line_numbers.py") + >>> import json + >>> json.dump(config, open("input.json", "w")) + + """ + code = load_transform_from_file(transform_file) + + return { + "code": code, + "input_filename": input_filename, + "output_filename": output_filename, + } + + +def validate_transform_function(file_path: str | Path) -> bool: + """Validate that a Python file contains a valid transform() function. + + Args: + file_path: Path to Python file to validate + + Returns: + True if valid, raises exception otherwise + + Raises: + ValueError: If transform() function is not found or invalid + + """ + code = load_transform_from_file(file_path) + + # Check if transform function is defined + if "def transform(" not in code: + raise ValueError( + f"File {file_path} must contain a 'def transform(data)' function" + ) + + # Try to compile the code + try: + compile(code, str(file_path), "exec") + except SyntaxError as e: + raise ValueError(f"Syntax error in {file_path}: {e}") from e + + # Try to execute and verify transform exists + namespace: dict[str, Any] = {"__builtins__": __builtins__} + try: + exec(code, namespace) + except Exception as e: + raise ValueError(f"Failed to execute {file_path}: {e}") from e + + if "transform" not in namespace: + raise ValueError(f"No 'transform' function found in {file_path}") + + if not callable(namespace["transform"]): + raise ValueError(f"'transform' in {file_path} is not callable") + + return True diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/default.py b/fuzzforge-common/src/fuzzforge_common/workflows/default.py new file mode 100644 index 0000000..b50e159 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/workflows/default.py @@ -0,0 +1,27 @@ +from fuzzforge_types import ( + FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime) + FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime) +) + +from fuzzforge_common.workflows.base.definitions import ( + FuzzForgeWorkflowDefinition, # noqa: TC001 (required by pydantic at runtime) +) +from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters + + +class ExecuteFuzzForgeWorkflowParameters(TemporalWorkflowParameters): + """Parameters for the default FuzzForge workflow orchestration. + + Contains workflow definition and execution tracking identifiers + for coordinating multi-module workflows. + + """ + + #: UUID7 identifier of this specific workflow execution. + execution_identifier: FuzzForgeExecutionIdentifier + + #: UUID7 identifier of the project this execution belongs to. + project_identifier: FuzzForgeProjectIdentifier + + #: The definition of the FuzzForge workflow to run. + workflow_definition: FuzzForgeWorkflowDefinition diff --git a/fuzzforge-common/src/fuzzforge_common/workflows/modules.py b/fuzzforge-common/src/fuzzforge_common/workflows/modules.py new file mode 100644 index 0000000..1573516 --- /dev/null +++ b/fuzzforge-common/src/fuzzforge_common/workflows/modules.py @@ -0,0 +1,80 @@ +from typing import Any, Literal + +from fuzzforge_types import ( + FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime) + FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime) +) + +from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters + + +class ExecuteFuzzForgeModuleParameters(TemporalWorkflowParameters): + """Parameters for executing a single FuzzForge module workflow. + + Contains module execution configuration including container image, + project context, and execution tracking identifiers. + + Supports workflow chaining where modules can be executed in sequence, + with each module's output becoming the next module's input. + + """ + + #: The identifier of this module execution. + execution_identifier: FuzzForgeExecutionIdentifier + + #: The identifier/name of the module to execute. + #: FIXME: Currently accepts both UUID (for registry lookups) and container names (e.g., "text-generator:0.0.1"). + #: This should be split into module_identifier (UUID) and container_image (string) in the future. + module_identifier: str + + #: The identifier of the project this module execution belongs to. + project_identifier: FuzzForgeProjectIdentifier + + #: Optional configuration dictionary for the module. + #: Will be written to /data/input/config.json in the sandbox. + module_configuration: dict[str, Any] | None = None + + # Workflow chaining fields + + #: The identifier of the parent workflow execution (if part of a multi-module workflow). + #: For standalone module executions, this equals execution_identifier. + workflow_execution_identifier: FuzzForgeExecutionIdentifier | None = None + + #: Position of this module in the workflow (0-based). + #: 0 = first module (reads from project assets) + #: N > 0 = subsequent module (reads from previous module's output) + step_index: int = 0 + + #: Execution identifier of the previous module in the workflow chain. + #: None for first module (step_index=0). + #: Used to locate previous module's output in storage. + previous_step_execution_identifier: FuzzForgeExecutionIdentifier | None = None + + +class WorkflowStep(TemporalWorkflowParameters): + """A step in a workflow - a module execution. + + Steps are executed sequentially in a workflow. Each step runs a containerized module. + + Examples: + # Module step + WorkflowStep( + step_index=0, + step_type="module", + module_identifier="text-generator:0.0.1" + ) + + """ + + #: Position of this step in the workflow (0-based) + step_index: int + + #: Type of step: "module" (bridges are also modules now) + step_type: Literal["module"] + + #: Module identifier (container image name like "text-generator:0.0.1") + #: Required if step_type="module" + module_identifier: str | None = None + + #: Optional module configuration + module_configuration: dict[str, Any] | None = None diff --git a/fuzzforge-common/tests/.gitkeep b/fuzzforge-common/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/tests/__init__.py b/fuzzforge-common/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/tests/conftest.py b/fuzzforge-common/tests/conftest.py new file mode 100644 index 0000000..f4324d8 --- /dev/null +++ b/fuzzforge-common/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ["fuzzforge_tests.fixtures"] diff --git a/fuzzforge-common/tests/unit/__init__.py b/fuzzforge-common/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/tests/unit/engines/__init__.py b/fuzzforge-common/tests/unit/engines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/tests/unit/engines/conftest.py b/fuzzforge-common/tests/unit/engines/conftest.py new file mode 100644 index 0000000..6baa337 --- /dev/null +++ b/fuzzforge-common/tests/unit/engines/conftest.py @@ -0,0 +1,9 @@ +import pytest + +from fuzzforge_common.sandboxes.engines.podman.engine import Podman + + +@pytest.fixture +def podman_engine(podman_socket: str) -> Podman: + """TODO.""" + return Podman(socket=podman_socket) diff --git a/fuzzforge-common/tests/unit/engines/test_podman.py b/fuzzforge-common/tests/unit/engines/test_podman.py new file mode 100644 index 0000000..e777039 --- /dev/null +++ b/fuzzforge-common/tests/unit/engines/test_podman.py @@ -0,0 +1,21 @@ +from typing import TYPE_CHECKING +from uuid import uuid4 + +if TYPE_CHECKING: + from pathlib import Path + + from podman import PodmanClient + + from fuzzforge_common.sandboxes.engines.podman.engine import Podman + + +def test_can_register_oci( + path_to_oci: Path, + podman_engine: Podman, + podman_client: PodmanClient, +) -> None: + """TODO.""" + repository: str = str(uuid4()) + podman_engine.register_archive(archive=path_to_oci, repository=repository) + assert podman_client.images.exists(key=repository) + podman_client.images.get(name=repository).remove() diff --git a/fuzzforge-common/tests/unit/storage/__init__.py b/fuzzforge-common/tests/unit/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-common/tests/unit/storage/test_storage.py b/fuzzforge-common/tests/unit/storage/test_storage.py new file mode 100644 index 0000000..9e97311 --- /dev/null +++ b/fuzzforge-common/tests/unit/storage/test_storage.py @@ -0,0 +1,42 @@ +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from fuzzforge_common.storage.configuration import StorageConfiguration + + +def test_download_directory( + storage_configuration: StorageConfiguration, + boto3_client: Any, + random_bucket: str, + tmp_path: Path, +) -> None: + """TODO.""" + bucket = random_bucket + storage = storage_configuration.into_storage() + + d1 = tmp_path.joinpath("d1") + f1 = d1.joinpath("f1") + d2 = tmp_path.joinpath("d2") + f2 = d2.joinpath("f2") + d3 = d2.joinpath("d3") + f3 = d3.joinpath("d3") + + d1.mkdir() + d2.mkdir() + d3.mkdir() + f1.touch() + f2.touch() + f3.touch() + + for path in [f1, f2, f3]: + key: Path = Path("assets", path.relative_to(other=tmp_path)) + boto3_client.upload_file( + Bucket=bucket, + Filename=str(path), + Key=str(key), + ) + + path = storage.download_directory(bucket=bucket, directory="assets") + + assert path.is_file() diff --git a/fuzzforge-mcp/Dockerfile b/fuzzforge-mcp/Dockerfile new file mode 100644 index 0000000..8d25f90 --- /dev/null +++ b/fuzzforge-mcp/Dockerfile @@ -0,0 +1,11 @@ +FROM docker.io/debian:trixie + +ARG PACKAGE + +COPY --from=ghcr.io/astral-sh/uv:0.9.10 /uv /bin/ + +WORKDIR /app + +RUN /bin/uv venv && /bin/uv pip install --find-links /wheels $PACKAGE + +CMD [ "/bin/uv", "run", "uvicorn", "fuzzforge_mcp.application:app"] diff --git a/fuzzforge-mcp/Makefile b/fuzzforge-mcp/Makefile new file mode 100644 index 0000000..3f37409 --- /dev/null +++ b/fuzzforge-mcp/Makefile @@ -0,0 +1,51 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +ARTIFACTS?=./dist +SOURCES=./src +TESTS=./tests + +.PHONY: bandit clean cloc format image mypy pytest ruff version wheel + +bandit: + uv run bandit --recursive $(SOURCES) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) $(TESTS) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +image: + $(PODMAN) build \ + --build-arg PACKAGE=$(PACKAGE) \ + --file ./Dockerfile \ + --no-cache \ + --tag $(PACKAGE):$(VERSION) \ + --volume $(ARTIFACTS):/wheels + +mypy: + uv run mypy $(SOURCES) $(TESTS) + +pytest: + uv run pytest -v $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' + +wheel: + uv build --out-dir $(ARTIFACTS) diff --git a/fuzzforge-mcp/README.md b/fuzzforge-mcp/README.md new file mode 100644 index 0000000..e86159c --- /dev/null +++ b/fuzzforge-mcp/README.md @@ -0,0 +1,223 @@ +# FuzzForge MCP + +Model Context Protocol (MCP) server that enables AI agents to orchestrate FuzzForge security research modules. + +## Overview + +FuzzForge MCP provides a standardized interface for AI agents (Claude Code, GitHub Copilot, Claude Desktop) to: + +- List and discover available security modules +- Execute modules in isolated containers +- Chain modules together in workflows +- Manage project assets and results + +The server communicates with AI agents using the [Model Context Protocol](https://modelcontextprotocol.io/) over stdio. + +## Installation + +### Automatic Installation (Recommended) + +Use the FuzzForge CLI to automatically configure MCP for your AI agent: + +```bash +# For GitHub Copilot +uv run fuzzforge mcp install copilot + +# For Claude Code (VS Code extension) +uv run fuzzforge mcp install claude-code + +# For Claude Desktop (standalone app) +uv run fuzzforge mcp install claude-desktop + +# Verify installation +uv run fuzzforge mcp status +``` + +After installation, restart your AI agent to activate the connection. + +### Manual Installation + +For custom setups, you can manually configure the MCP server. + +#### Claude Code (`.mcp.json` in project root) + +```json +{ + "mcpServers": { + "fuzzforge": { + "command": "/path/to/fuzzforge-oss/.venv/bin/python", + "args": ["-m", "fuzzforge_mcp"], + "cwd": "/path/to/fuzzforge-oss", + "env": { + "FUZZFORGE_MODULES_PATH": "/path/to/fuzzforge-oss/fuzzforge-modules", + "FUZZFORGE_ENGINE__TYPE": "podman", + "FUZZFORGE_ENGINE__GRAPHROOT": "~/.fuzzforge/containers/storage", + "FUZZFORGE_ENGINE__RUNROOT": "~/.fuzzforge/containers/run" + } + } + } +} +``` + +#### GitHub Copilot (`~/.config/Code/User/mcp.json`) + +```json +{ + "servers": { + "fuzzforge": { + "type": "stdio", + "command": "/path/to/fuzzforge-oss/.venv/bin/python", + "args": ["-m", "fuzzforge_mcp"], + "cwd": "/path/to/fuzzforge-oss", + "env": { + "FUZZFORGE_MODULES_PATH": "/path/to/fuzzforge-oss/fuzzforge-modules", + "FUZZFORGE_ENGINE__TYPE": "podman", + "FUZZFORGE_ENGINE__GRAPHROOT": "~/.fuzzforge/containers/storage", + "FUZZFORGE_ENGINE__RUNROOT": "~/.fuzzforge/containers/run" + } + } + } +} +``` + +#### Claude Desktop (`~/.config/Claude/claude_desktop_config.json`) + +```json +{ + "mcpServers": { + "fuzzforge": { + "type": "stdio", + "command": "/path/to/fuzzforge-oss/.venv/bin/python", + "args": ["-m", "fuzzforge_mcp"], + "cwd": "/path/to/fuzzforge-oss", + "env": { + "FUZZFORGE_MODULES_PATH": "/path/to/fuzzforge-oss/fuzzforge-modules", + "FUZZFORGE_ENGINE__TYPE": "podman", + "FUZZFORGE_ENGINE__GRAPHROOT": "~/.fuzzforge/containers/storage", + "FUZZFORGE_ENGINE__RUNROOT": "~/.fuzzforge/containers/run" + } + } + } +} +``` + +## Environment Variables + +| Variable | Required | Default | Description | +| -------- | -------- | ------- | ----------- | +| `FUZZFORGE_MODULES_PATH` | Yes | - | Path to the modules directory | +| `FUZZFORGE_ENGINE__TYPE` | No | `podman` | Container engine (`podman` or `docker`) | +| `FUZZFORGE_ENGINE__GRAPHROOT` | No | `~/.fuzzforge/containers/storage` | Container image storage path | +| `FUZZFORGE_ENGINE__RUNROOT` | No | `~/.fuzzforge/containers/run` | Container runtime state path | + +## Available Tools + +The MCP server exposes the following tools to AI agents: + +### Project Management + +- **`init_project`** - Initialize a new FuzzForge project +- **`set_project_assets`** - Set initial assets (source code, contracts, etc.) for the project + +### Module Management + +- **`list_modules`** - List all available security research modules +- **`execute_module`** - Execute a single module in an isolated container + +### Workflow Management + +- **`execute_workflow`** - Execute a workflow consisting of multiple chained modules + +### Resources + +The server also provides resources for accessing: + +- Project information and configuration +- Module metadata and schemas +- Execution results and artifacts +- Workflow definitions and status + +## Usage Examples + +### From AI Agent (e.g., Claude Code) + +Once configured, AI agents can interact with FuzzForge naturally: + +```text +User: List the available security modules + +AI Agent: [Calls list_modules tool] +``` + +```text +User: Run echidna fuzzer on my Solidity contracts + +AI Agent: [Calls init_project, set_project_assets, then execute_module] +``` + +```text +User: Create a workflow that compiles contracts, runs slither, then echidna + +AI Agent: [Calls execute_workflow with appropriate steps] +``` + +### Direct Testing (Development) + +For testing during development, you can run the MCP server directly: + +```bash +# Run MCP server in stdio mode (for AI agents) +uv run python -m fuzzforge_mcp + +# Run HTTP server for testing (not for production) +uv run uvicorn fuzzforge_mcp.application:app --reload +``` + +## Architecture + +```text +┌─────────────────────────────────────────┐ +│ AI Agent (Claude/Copilot) │ +│ via MCP Protocol │ +└─────────────────────────────────────────┘ + │ + │ stdio/JSON-RPC + ▼ +┌─────────────────────────────────────────┐ +│ FuzzForge MCP Server │ +│ Tools: init_project, list_modules, │ +│ execute_module, execute_workflow│ +└─────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────┐ +│ FuzzForge Runner │ +│ Podman/Docker Orchestration │ +└─────────────────────────────────────────┘ + │ + ┌─────────┼─────────┐ + ▼ ▼ ▼ + [Module 1] [Module 2] [Module 3] + Container Container Container +``` + +## Development + +### Building the Package + +```bash +# Install development dependencies +uv sync + +# Run type checking +uv run mypy src/ + +# Run tests +uv run pytest +``` + +## See Also + +- [FuzzForge Main README](../README.md) - Overall project documentation +- [Module SDK](../fuzzforge-modules/fuzzforge-modules-sdk/README.md) - Creating custom modules +- [Model Context Protocol](https://modelcontextprotocol.io/) - MCP specification diff --git a/fuzzforge-mcp/mypy.ini b/fuzzforge-mcp/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-mcp/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-mcp/pyproject.toml b/fuzzforge-mcp/pyproject.toml new file mode 100644 index 0000000..1458a28 --- /dev/null +++ b/fuzzforge-mcp/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "fuzzforge-mcp" +version = "0.0.1" +description = "FuzzForge MCP Server - AI agent gateway for FuzzForge OSS." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fastmcp==2.14.1", + "fuzzforge-runner==0.0.1", + "fuzzforge-types==0.0.1", + "pydantic==2.12.4", + "pydantic-settings==2.12.0", + "structlog==25.5.0", +] + +[project.scripts] +fuzzforge-mcp = "fuzzforge_mcp.__main__:main" + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", + "pytest-asyncio==1.3.0", + "pytest-httpx==0.36.0", +] + +[tool.uv.sources] +fuzzforge-runner = { workspace = true } +fuzzforge-types = { workspace = true } diff --git a/fuzzforge-mcp/pytest.ini b/fuzzforge-mcp/pytest.ini new file mode 100644 index 0000000..2f4c80e --- /dev/null +++ b/fuzzforge-mcp/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto diff --git a/fuzzforge-mcp/ruff.toml b/fuzzforge-mcp/ruff.toml new file mode 100644 index 0000000..c3310b5 --- /dev/null +++ b/fuzzforge-mcp/ruff.toml @@ -0,0 +1,16 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR0913", # allowing functions with many arguments in tests (required for fixtures) + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/__init__.py b/fuzzforge-mcp/src/fuzzforge_mcp/__init__.py new file mode 100644 index 0000000..4578657 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/__init__.py @@ -0,0 +1 @@ +"""TODO.""" diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/__main__.py b/fuzzforge-mcp/src/fuzzforge_mcp/__main__.py new file mode 100644 index 0000000..a740b16 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/__main__.py @@ -0,0 +1,17 @@ +"""FuzzForge MCP Server entry point.""" + +from fuzzforge_mcp.application import mcp + + +def main() -> None: + """Run the FuzzForge MCP server in stdio mode. + + This is the primary entry point for AI agent integration. + The server communicates via stdin/stdout using the MCP protocol. + + """ + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/application.py b/fuzzforge-mcp/src/fuzzforge_mcp/application.py new file mode 100644 index 0000000..501ed02 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/application.py @@ -0,0 +1,64 @@ +"""FuzzForge MCP Server Application. + +This is the main entry point for the FuzzForge MCP server, providing +AI agents with tools to execute security research modules. + +""" + +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +from fastmcp import FastMCP +from fastmcp.server.middleware.error_handling import ErrorHandlingMiddleware + +from fuzzforge_mcp import resources, tools +from fuzzforge_runner import Settings + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + +@asynccontextmanager +async def lifespan(_: FastMCP) -> AsyncGenerator[Settings]: + """Initialize MCP server lifespan context. + + Loads settings from environment variables and makes them + available to all tools and resources. + + :param mcp: FastMCP server instance (unused). + :return: Settings instance for dependency injection. + + """ + settings: Settings = Settings() + yield settings + + +mcp: FastMCP = FastMCP( + name="FuzzForge MCP Server", + instructions=""" +FuzzForge is a security research orchestration platform. Use these tools to: + +1. **List modules**: Discover available security research modules +2. **Execute modules**: Run modules in isolated containers +3. **Execute workflows**: Chain multiple modules together +4. **Manage projects**: Initialize and configure projects +5. **Get results**: Retrieve execution results + +Typical workflow: +1. Initialize a project with `init_project` +2. Set project assets with `set_project_assets` (optional) +3. List available modules with `list_modules` +4. Execute a module with `execute_module` +5. Get results with `get_execution_results` +""", + lifespan=lifespan, +) + +mcp.add_middleware(middleware=ErrorHandlingMiddleware()) + +mcp.mount(resources.mcp) +mcp.mount(tools.mcp) + +# HTTP app for testing (primary mode is stdio) +app = mcp.http_app() + diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/dependencies.py b/fuzzforge-mcp/src/fuzzforge_mcp/dependencies.py new file mode 100644 index 0000000..4339c86 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/dependencies.py @@ -0,0 +1,48 @@ +"""Dependency injection helpers for FuzzForge MCP.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from fastmcp.server.dependencies import get_context +from fuzzforge_runner import Runner, Settings + +from fuzzforge_mcp.exceptions import FuzzForgeMCPError + +if TYPE_CHECKING: + from fastmcp import Context + + +def get_settings() -> Settings: + """Get MCP server settings from context. + + :return: Settings instance. + :raises FuzzForgeMCPError: If settings not available. + + """ + context: Context = get_context() + if context.request_context is None: + message: str = "Request context not available" + raise FuzzForgeMCPError(message) + return cast("Settings", context.request_context.lifespan_context) + + +def get_project_path() -> Path: + """Get the current project path. + + :return: Path to the current project. + + """ + settings: Settings = get_settings() + return Path(settings.project.default_path) + + +def get_runner() -> Runner: + """Get a configured Runner instance. + + :return: Runner instance configured from MCP settings. + + """ + settings: Settings = get_settings() + return Runner(settings) diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/exceptions.py b/fuzzforge-mcp/src/fuzzforge_mcp/exceptions.py new file mode 100644 index 0000000..527739f --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/exceptions.py @@ -0,0 +1,5 @@ +"""TODO.""" + + +class FuzzForgeMCPError(Exception): + """TODO.""" diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/resources/__init__.py b/fuzzforge-mcp/src/fuzzforge_mcp/resources/__init__.py new file mode 100644 index 0000000..f6a1ce0 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/resources/__init__.py @@ -0,0 +1,16 @@ +"""FuzzForge MCP Resources.""" + +from fastmcp import FastMCP + +from fuzzforge_mcp.resources import executions, modules, project, workflows + +mcp: FastMCP = FastMCP() + +mcp.mount(executions.mcp) +mcp.mount(modules.mcp) +mcp.mount(project.mcp) +mcp.mount(workflows.mcp) + +__all__ = [ + "mcp", +] diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/resources/executions.py b/fuzzforge-mcp/src/fuzzforge_mcp/resources/executions.py new file mode 100644 index 0000000..c3d5f31 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/resources/executions.py @@ -0,0 +1,75 @@ +"""Execution resources for FuzzForge MCP.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fastmcp import FastMCP +from fastmcp.exceptions import ResourceError + +from fuzzforge_mcp.dependencies import get_project_path, get_runner + +if TYPE_CHECKING: + from fuzzforge_runner import Runner + + +mcp: FastMCP = FastMCP() + + +@mcp.resource("fuzzforge://executions/") +async def list_executions() -> list[dict[str, Any]]: + """List all executions for the current project. + + Returns a list of execution IDs and basic metadata. + + :return: List of execution information dictionaries. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + execution_ids = runner.list_executions(project_path) + + return [ + { + "execution_id": exec_id, + "has_results": runner.get_execution_results(project_path, exec_id) is not None, + } + for exec_id in execution_ids + ] + + except Exception as exception: + message: str = f"Failed to list executions: {exception}" + raise ResourceError(message) from exception + + +@mcp.resource("fuzzforge://executions/{execution_id}") +async def get_execution(execution_id: str) -> dict[str, Any]: + """Get information about a specific execution. + + :param execution_id: The execution ID to retrieve. + :return: Execution information dictionary. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + results_path = runner.get_execution_results(project_path, execution_id) + + if results_path is None: + raise ResourceError(f"Execution not found: {execution_id}") + + return { + "execution_id": execution_id, + "results_path": str(results_path), + "results_exist": results_path.exists(), + } + + except ResourceError: + raise + except Exception as exception: + message: str = f"Failed to get execution: {exception}" + raise ResourceError(message) from exception diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/resources/modules.py b/fuzzforge-mcp/src/fuzzforge_mcp/resources/modules.py new file mode 100644 index 0000000..a551ccd --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/resources/modules.py @@ -0,0 +1,78 @@ +"""Module resources for FuzzForge MCP.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from fastmcp import FastMCP +from fastmcp.exceptions import ResourceError + +from fuzzforge_mcp.dependencies import get_runner + +if TYPE_CHECKING: + from fuzzforge_runner import Runner + from fuzzforge_runner.runner import ModuleInfo + + +mcp: FastMCP = FastMCP() + + +@mcp.resource("fuzzforge://modules/") +async def list_modules() -> list[dict[str, Any]]: + """List all available FuzzForge modules. + + Returns information about modules that can be executed, + including their identifiers and availability status. + + :return: List of module information dictionaries. + + """ + runner: Runner = get_runner() + + try: + modules: list[ModuleInfo] = runner.list_modules() + + return [ + { + "identifier": module.identifier, + "description": module.description, + "version": module.version, + "available": module.available, + } + for module in modules + ] + + except Exception as exception: + message: str = f"Failed to list modules: {exception}" + raise ResourceError(message) from exception + + +@mcp.resource("fuzzforge://modules/{module_identifier}") +async def get_module(module_identifier: str) -> dict[str, Any]: + """Get information about a specific module. + + :param module_identifier: The identifier of the module to retrieve. + :return: Module information dictionary. + + """ + runner: Runner = get_runner() + + try: + module: ModuleInfo | None = runner.get_module_info(module_identifier) + + if module is None: + raise ResourceError(f"Module not found: {module_identifier}") + + return { + "identifier": module.identifier, + "description": module.description, + "version": module.version, + "available": module.available, + } + + except ResourceError: + raise + except Exception as exception: + message: str = f"Failed to get module: {exception}" + raise ResourceError(message) from exception + diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/resources/project.py b/fuzzforge-mcp/src/fuzzforge_mcp/resources/project.py new file mode 100644 index 0000000..566ba0f --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/resources/project.py @@ -0,0 +1,84 @@ +"""Project resources for FuzzForge MCP.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fastmcp import FastMCP +from fastmcp.exceptions import ResourceError + +from fuzzforge_mcp.dependencies import get_project_path, get_runner + +if TYPE_CHECKING: + from fuzzforge_runner import Runner + + +mcp: FastMCP = FastMCP() + + +@mcp.resource("fuzzforge://project") +async def get_project() -> dict[str, Any]: + """Get information about the current project. + + Returns the current project configuration including paths + and available executions. + + :return: Project information dictionary. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + executions = runner.list_executions(project_path) + assets_path = runner.storage.get_project_assets_path(project_path) + + return { + "path": str(project_path), + "name": project_path.name, + "has_assets": assets_path is not None, + "assets_path": str(assets_path) if assets_path else None, + "execution_count": len(executions), + "recent_executions": executions[:10], # Last 10 executions + } + + except Exception as exception: + message: str = f"Failed to get project info: {exception}" + raise ResourceError(message) from exception + + +@mcp.resource("fuzzforge://project/settings") +async def get_project_settings() -> dict[str, Any]: + """Get current FuzzForge settings. + + Returns the active configuration for the MCP server including + engine, storage, and project settings. + + :return: Settings dictionary. + + """ + from fuzzforge_mcp.dependencies import get_settings + + try: + settings = get_settings() + + return { + "engine": { + "type": settings.engine.type, + "socket": settings.engine.socket, + }, + "storage": { + "path": str(settings.storage.path), + }, + "project": { + "path": str(settings.project.path), + "modules_path": str(settings.modules_path), + }, + "debug": settings.debug, + } + + except Exception as exception: + message: str = f"Failed to get settings: {exception}" + raise ResourceError(message) from exception + diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/resources/workflows.py b/fuzzforge-mcp/src/fuzzforge_mcp/resources/workflows.py new file mode 100644 index 0000000..968dce9 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/resources/workflows.py @@ -0,0 +1,53 @@ +"""Workflow resources for FuzzForge MCP. + +Note: In FuzzForge OSS, workflows are defined at runtime rather than +stored. This resource provides documentation about workflow capabilities. + +""" + +from __future__ import annotations + +from typing import Any + +from fastmcp import FastMCP + + +mcp: FastMCP = FastMCP() + + +@mcp.resource("fuzzforge://workflows/help") +async def get_workflow_help() -> dict[str, Any]: + """Get help information about creating workflows. + + Workflows in FuzzForge OSS are defined at execution time rather + than stored. Use the execute_workflow tool with step definitions. + + :return: Workflow documentation. + + """ + return { + "description": "Workflows chain multiple modules together", + "usage": "Use the execute_workflow tool with step definitions", + "example": { + "workflow_name": "security-audit", + "steps": [ + { + "module": "compile-contracts", + "configuration": {"solc_version": "0.8.0"}, + }, + { + "module": "slither", + "configuration": {}, + }, + { + "module": "echidna", + "configuration": {"test_limit": 10000}, + }, + ], + }, + "step_format": { + "module": "Module identifier (required)", + "configuration": "Module-specific configuration (optional)", + "name": "Step name for logging (optional)", + }, + } diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/tools/__init__.py b/fuzzforge-mcp/src/fuzzforge_mcp/tools/__init__.py new file mode 100644 index 0000000..3a9edc9 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/tools/__init__.py @@ -0,0 +1,16 @@ +"""FuzzForge MCP Tools.""" + +from fastmcp import FastMCP + +from fuzzforge_mcp.tools import modules, projects, workflows + +mcp: FastMCP = FastMCP() + +mcp.mount(modules.mcp) +mcp.mount(projects.mcp) +mcp.mount(workflows.mcp) + +__all__ = [ + "mcp", +] + diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/tools/modules.py b/fuzzforge-mcp/src/fuzzforge_mcp/tools/modules.py new file mode 100644 index 0000000..d38e820 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/tools/modules.py @@ -0,0 +1,347 @@ +"""Module tools for FuzzForge MCP.""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fastmcp import FastMCP +from fastmcp.exceptions import ToolError + +from fuzzforge_mcp.dependencies import get_project_path, get_runner, get_settings + +if TYPE_CHECKING: + from fuzzforge_runner import Runner + from fuzzforge_runner.orchestrator import StepResult + + +mcp: FastMCP = FastMCP() + +# Track running background executions +_background_executions: dict[str, dict[str, Any]] = {} + + +@mcp.tool +async def list_modules() -> dict[str, Any]: + """List all available FuzzForge modules. + + Returns information about modules that can be executed, + including their identifiers and availability status. + + :return: Dictionary with list of available modules and their details. + + """ + try: + runner: Runner = get_runner() + settings = get_settings() + + # Use the engine abstraction to list images + modules = runner.list_module_images(filter_prefix="localhost/") + + available_modules = [ + { + "identifier": module.identifier, + "image": f"localhost/{module.identifier}:{module.version or 'latest'}", + "available": module.available, + } + for module in modules + ] + + return { + "modules": available_modules, + "count": len(available_modules), + "container_engine": settings.engine.type, + "registry_url": settings.registry.url, + "registry_tag": settings.registry.default_tag, + } + + except Exception as exception: + message: str = f"Failed to list modules: {exception}" + raise ToolError(message) from exception + + +@mcp.tool +async def execute_module( + module_identifier: str, + configuration: dict[str, Any] | None = None, + assets_path: str | None = None, +) -> dict[str, Any]: + """Execute a FuzzForge module in an isolated container. + + This tool runs a module in a sandboxed environment. + The module receives input assets and produces output results. + + :param module_identifier: The identifier of the module to execute. + :param configuration: Optional configuration dict to pass to the module. + :param assets_path: Optional path to input assets. If not provided, uses project assets. + :return: Execution result including status and results path. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + result: StepResult = await runner.execute_module( + module_identifier=module_identifier, + project_path=project_path, + configuration=configuration, + assets_path=Path(assets_path) if assets_path else None, + ) + + return { + "success": result.success, + "execution_id": result.execution_id, + "module": result.module_identifier, + "results_path": str(result.results_path) if result.results_path else None, + "started_at": result.started_at.isoformat(), + "completed_at": result.completed_at.isoformat(), + "error": result.error, + } + + except Exception as exception: + message: str = f"Module execution failed: {exception}" + raise ToolError(message) from exception + + +@mcp.tool +async def start_continuous_module( + module_identifier: str, + configuration: dict[str, Any] | None = None, + assets_path: str | None = None, +) -> dict[str, Any]: + """Start a module in continuous/background mode. + + The module will run indefinitely until stopped with stop_continuous_module(). + Use get_continuous_status() to check progress and metrics. + + This is useful for long-running modules that should run until + the user decides to stop them. + + :param module_identifier: The module to run. + :param configuration: Optional configuration. Set max_duration to 0 for infinite. + :param assets_path: Optional path to input assets. + :return: Execution info including session_id for monitoring. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + session_id = str(uuid.uuid4())[:8] + + # Set infinite duration if not specified + if configuration is None: + configuration = {} + if "max_duration" not in configuration: + configuration["max_duration"] = 0 # 0 = infinite + + try: + # Determine assets path + if assets_path: + actual_assets_path = Path(assets_path) + else: + storage = runner.storage + actual_assets_path = storage.get_project_assets_path(project_path) + + # Use the new non-blocking executor method + executor = runner._executor + result = executor.start_module_continuous( + module_identifier=module_identifier, + assets_path=actual_assets_path, + configuration=configuration, + ) + + # Store execution info for tracking + _background_executions[session_id] = { + "session_id": session_id, + "module": module_identifier, + "configuration": configuration, + "started_at": datetime.now(timezone.utc).isoformat(), + "status": "running", + "container_id": result["container_id"], + "input_dir": result["input_dir"], + } + + return { + "success": True, + "session_id": session_id, + "module": module_identifier, + "container_id": result["container_id"], + "status": "running", + "message": f"Continuous module started. Use get_continuous_status('{session_id}') to monitor progress.", + } + + except Exception as exception: + message: str = f"Failed to start continuous module: {exception}" + raise ToolError(message) from exception + + +def _get_continuous_status_impl(session_id: str) -> dict[str, Any]: + """Internal helper to get continuous session status (non-tool version).""" + if session_id not in _background_executions: + raise ToolError(f"Unknown session: {session_id}. Use list_continuous_sessions() to see active sessions.") + + execution = _background_executions[session_id] + container_id = execution.get("container_id") + + # Initialize metrics + metrics: dict[str, Any] = { + "total_executions": 0, + "total_crashes": 0, + "exec_per_sec": 0, + "coverage": 0, + "current_target": "", + "latest_events": [], + } + + # Read stream.jsonl from inside the running container + if container_id: + try: + runner: Runner = get_runner() + executor = runner._executor + + # Check container status first + container_status = executor.get_module_status(container_id) + if container_status != "running": + execution["status"] = "stopped" if container_status == "exited" else container_status + + # Read stream.jsonl from container + stream_content = executor.read_module_output(container_id, "/data/output/stream.jsonl") + + if stream_content: + lines = stream_content.strip().split("\n") + # Get last 20 events + recent_lines = lines[-20:] if len(lines) > 20 else lines + crash_count = 0 + + for line in recent_lines: + try: + event = json.loads(line) + metrics["latest_events"].append(event) + + # Extract metrics from events + if event.get("event") == "metrics": + metrics["total_executions"] = event.get("executions", 0) + metrics["current_target"] = event.get("target", "") + metrics["exec_per_sec"] = event.get("exec_per_sec", 0) + metrics["coverage"] = event.get("coverage", 0) + + if event.get("event") == "crash_detected": + crash_count += 1 + + except json.JSONDecodeError: + continue + + metrics["total_crashes"] = crash_count + + except Exception as e: + metrics["error"] = str(e) + + # Calculate elapsed time + started_at = execution.get("started_at", "") + elapsed_seconds = 0 + if started_at: + try: + start_time = datetime.fromisoformat(started_at) + elapsed_seconds = int((datetime.now(timezone.utc) - start_time).total_seconds()) + except Exception: + pass + + return { + "session_id": session_id, + "module": execution.get("module"), + "status": execution.get("status"), + "container_id": container_id, + "started_at": started_at, + "elapsed_seconds": elapsed_seconds, + "elapsed_human": f"{elapsed_seconds // 60}m {elapsed_seconds % 60}s", + "metrics": metrics, + } + + +@mcp.tool +async def get_continuous_status(session_id: str) -> dict[str, Any]: + """Get the current status and metrics of a running continuous session. + + Call this periodically (e.g., every 30 seconds) to get live updates + on progress and metrics. + + :param session_id: The session ID returned by start_continuous_module(). + :return: Current status, metrics, and any events found. + + """ + return _get_continuous_status_impl(session_id) + + +@mcp.tool +async def stop_continuous_module(session_id: str) -> dict[str, Any]: + """Stop a running continuous session. + + This will gracefully stop the module and collect any results. + + :param session_id: The session ID of the session to stop. + :return: Final status and summary of the session. + + """ + if session_id not in _background_executions: + raise ToolError(f"Unknown session: {session_id}") + + execution = _background_executions[session_id] + container_id = execution.get("container_id") + input_dir = execution.get("input_dir") + + try: + # Get final metrics before stopping (use helper, not the tool) + final_metrics = _get_continuous_status_impl(session_id) + + # Stop the container and collect results + results_path = None + if container_id: + runner: Runner = get_runner() + executor = runner._executor + + try: + results_path = executor.stop_module_continuous(container_id, input_dir) + except Exception: + # Container may have already stopped + pass + + execution["status"] = "stopped" + execution["stopped_at"] = datetime.now(timezone.utc).isoformat() + + return { + "success": True, + "session_id": session_id, + "message": "Continuous session stopped", + "results_path": str(results_path) if results_path else None, + "final_metrics": final_metrics.get("metrics", {}), + "elapsed": final_metrics.get("elapsed_human", ""), + } + + except Exception as exception: + message: str = f"Failed to stop continuous module: {exception}" + raise ToolError(message) from exception + + +@mcp.tool +async def list_continuous_sessions() -> dict[str, Any]: + """List all active and recent continuous sessions. + + :return: List of continuous sessions with their status. + + """ + sessions = [] + for session_id, execution in _background_executions.items(): + sessions.append({ + "session_id": session_id, + "module": execution.get("module"), + "status": execution.get("status"), + "started_at": execution.get("started_at"), + }) + + return { + "sessions": sessions, + "count": len(sessions), + } + diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/tools/projects.py b/fuzzforge-mcp/src/fuzzforge_mcp/tools/projects.py new file mode 100644 index 0000000..44c4f6b --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/tools/projects.py @@ -0,0 +1,145 @@ +"""Project management tools for FuzzForge MCP.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fastmcp import FastMCP +from fastmcp.exceptions import ToolError + +from fuzzforge_mcp.dependencies import get_project_path, get_runner + +if TYPE_CHECKING: + from fuzzforge_runner import Runner + + +mcp: FastMCP = FastMCP() + + +@mcp.tool +async def init_project(project_path: str | None = None) -> dict[str, Any]: + """Initialize a new FuzzForge project. + + Creates the necessary storage directories for a project. This should + be called before executing modules or workflows. + + :param project_path: Path to the project directory. If not provided, uses current directory. + :return: Project initialization result. + + """ + runner: Runner = get_runner() + + try: + path = Path(project_path) if project_path else get_project_path() + storage_path = runner.init_project(path) + + return { + "success": True, + "project_path": str(path), + "storage_path": str(storage_path), + "message": f"Project initialized at {path}", + } + + except Exception as exception: + message: str = f"Failed to initialize project: {exception}" + raise ToolError(message) from exception + + +@mcp.tool +async def set_project_assets(assets_path: str) -> dict[str, Any]: + """Set the initial assets for a project. + + Assets are input files that will be provided to modules during execution. + This could be source code, contracts, binaries, etc. + + :param assets_path: Path to assets file (archive) or directory. + :return: Result including stored assets path. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + stored_path = runner.set_project_assets( + project_path=project_path, + assets_path=Path(assets_path), + ) + + return { + "success": True, + "project_path": str(project_path), + "assets_path": str(stored_path), + "message": f"Assets stored from {assets_path}", + } + + except Exception as exception: + message: str = f"Failed to set project assets: {exception}" + raise ToolError(message) from exception + + +@mcp.tool +async def list_executions() -> dict[str, Any]: + """List all executions for the current project. + + Returns a list of execution IDs that can be used to retrieve results. + + :return: List of execution IDs. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + executions = runner.list_executions(project_path) + + return { + "success": True, + "project_path": str(project_path), + "executions": executions, + "count": len(executions), + } + + except Exception as exception: + message: str = f"Failed to list executions: {exception}" + raise ToolError(message) from exception + + +@mcp.tool +async def get_execution_results(execution_id: str, extract_to: str | None = None) -> dict[str, Any]: + """Get results for a specific execution. + + :param execution_id: The execution ID to retrieve results for. + :param extract_to: Optional directory to extract results to. + :return: Result including path to results archive. + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + results_path = runner.get_execution_results(project_path, execution_id) + + if results_path is None: + return { + "success": False, + "execution_id": execution_id, + "error": "Execution results not found", + } + + result = { + "success": True, + "execution_id": execution_id, + "results_path": str(results_path), + } + + # Extract if requested + if extract_to: + extracted_path = runner.extract_results(results_path, Path(extract_to)) + result["extracted_path"] = str(extracted_path) + + return result + + except Exception as exception: + message: str = f"Failed to get execution results: {exception}" + raise ToolError(message) from exception diff --git a/fuzzforge-mcp/src/fuzzforge_mcp/tools/workflows.py b/fuzzforge-mcp/src/fuzzforge_mcp/tools/workflows.py new file mode 100644 index 0000000..222ca60 --- /dev/null +++ b/fuzzforge-mcp/src/fuzzforge_mcp/tools/workflows.py @@ -0,0 +1,92 @@ +"""Workflow tools for FuzzForge MCP.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from fastmcp import FastMCP +from fastmcp.exceptions import ToolError +from fuzzforge_runner.orchestrator import WorkflowDefinition, WorkflowStep + +from fuzzforge_mcp.dependencies import get_project_path, get_runner + +if TYPE_CHECKING: + from fuzzforge_runner import Runner + from fuzzforge_runner.orchestrator import WorkflowResult + + +mcp: FastMCP = FastMCP() + + +@mcp.tool +async def execute_workflow( + workflow_name: str, + steps: list[dict[str, Any]], + initial_assets_path: str | None = None, +) -> dict[str, Any]: + """Execute a workflow consisting of multiple module steps. + + A workflow chains multiple modules together, passing the output of each + module as input to the next. This enables complex pipelines. + + :param workflow_name: Name for this workflow execution. + :param steps: List of step definitions, each with "module" and optional "configuration". + :param initial_assets_path: Optional path to initial assets for the first step. + :return: Workflow execution result including status of each step. + + Example steps format: + [ + {"module": "module-a", "configuration": {"key": "value"}}, + {"module": "module-b", "configuration": {}}, + {"module": "module-c"} + ] + + """ + runner: Runner = get_runner() + project_path: Path = get_project_path() + + try: + # Convert step dicts to WorkflowStep objects + workflow_steps = [ + WorkflowStep( + module_identifier=step["module"], + configuration=step.get("configuration"), + name=step.get("name", f"step-{i}"), + ) + for i, step in enumerate(steps) + ] + + workflow = WorkflowDefinition( + name=workflow_name, + steps=workflow_steps, + ) + + result: WorkflowResult = await runner.execute_workflow( + workflow=workflow, + project_path=project_path, + initial_assets_path=Path(initial_assets_path) if initial_assets_path else None, + ) + + return { + "success": result.success, + "execution_id": result.execution_id, + "workflow_name": result.name, + "final_results_path": str(result.final_results_path) if result.final_results_path else None, + "steps": [ + { + "step_index": step.step_index, + "module": step.module_identifier, + "success": step.success, + "execution_id": step.execution_id, + "results_path": str(step.results_path) if step.results_path else None, + "error": step.error, + } + for step in result.steps + ], + } + + except Exception as exception: + message: str = f"Workflow execution failed: {exception}" + raise ToolError(message) from exception + diff --git a/fuzzforge-mcp/tests/__init__.py b/fuzzforge-mcp/tests/__init__.py new file mode 100644 index 0000000..4578657 --- /dev/null +++ b/fuzzforge-mcp/tests/__init__.py @@ -0,0 +1 @@ +"""TODO.""" diff --git a/fuzzforge-mcp/tests/conftest.py b/fuzzforge-mcp/tests/conftest.py new file mode 100644 index 0000000..988ac7d --- /dev/null +++ b/fuzzforge-mcp/tests/conftest.py @@ -0,0 +1,34 @@ +"""TODO.""" + +from typing import TYPE_CHECKING + +import pytest +from fastmcp import Client + +from fuzzforge_mcp.application import mcp + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Callable + + from fastmcp.client import FastMCPTransport + from fuzzforge_types import FuzzForgeProjectIdentifier + +pytest_plugins = ["fuzzforge_tests.fixtures"] + + +@pytest.fixture(autouse=True) +def environment( + monkeypatch: pytest.MonkeyPatch, + random_project_identifier: Callable[[], FuzzForgeProjectIdentifier], +) -> None: + """TODO.""" + monkeypatch.setenv("FUZZFORGE_PROJECT_IDENTIFIER", str(random_project_identifier())) + monkeypatch.setenv("FUZZFORGE_API_HOST", "127.0.0.1") + monkeypatch.setenv("FUZZFORGE_API_PORT", "8000") + + +@pytest.fixture +async def mcp_client() -> AsyncGenerator[Client[FastMCPTransport]]: + """TODO.""" + async with Client(transport=mcp) as client: + yield client diff --git a/fuzzforge-mcp/tests/test_resources.py b/fuzzforge-mcp/tests/test_resources.py new file mode 100644 index 0000000..1924b8d --- /dev/null +++ b/fuzzforge-mcp/tests/test_resources.py @@ -0,0 +1,189 @@ +"""TODO.""" + +from datetime import UTC, datetime +from http import HTTPMethod +from json import loads +from typing import TYPE_CHECKING, cast + +from fuzzforge_sdk.api.responses.executions import ( + GetFuzzForgeModuleExecutionsResponse, + GetFuzzForgeWorkflowExecutionsResponse, + ModuleExecutionSummary, + WorkflowExecutionSummary, +) +from fuzzforge_sdk.api.responses.modules import ( + GetFuzzForgeModuleDefinitionResponse, + GetFuzzForgeModuleDefinitionsResponse, + ModuleDefinitionSummary, +) +from fuzzforge_types import FuzzForgeExecutionStatus, FuzzForgeModule + +if TYPE_CHECKING: + from collections.abc import Callable + + from fastmcp import Client + from fastmcp.client import FastMCPTransport + from fuzzforge_types import ( + FuzzForgeExecutionIdentifier, + FuzzForgeModuleIdentifier, + FuzzForgeProjectIdentifier, + FuzzForgeWorkflowIdentifier, + ) + from mcp.types import TextResourceContents + from pytest_httpx import HTTPXMock + + +async def test_get_fuzzforge_module_when_server_returns_valid_data( + httpx_mock: HTTPXMock, + mcp_client: Client[FastMCPTransport], + random_module_description: Callable[[], str], + random_module_identifier: Callable[[], FuzzForgeModuleIdentifier], + random_module_name: Callable[[], str], +) -> None: + """TODO.""" + module = GetFuzzForgeModuleDefinitionResponse( + module_description=random_module_description(), + module_identifier=random_module_identifier(), + module_name=random_module_name(), + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ) + httpx_mock.add_response( + json=module.model_dump(mode="json"), + method=HTTPMethod.GET, + url=f"http://127.0.0.1:8000/modules/{module.module_identifier}", + ) + response = FuzzForgeModule.model_validate_json( + json_data=cast( + "TextResourceContents", + (await mcp_client.read_resource(f"fuzzforge://modules/{module.module_identifier}"))[0], + ).text, + ) + assert response.module_description == module.module_description + assert response.module_identifier == module.module_identifier + assert response.module_name == module.module_name + + +async def test_get_fuzzforge_modules_when_server_returns_valid_data( + httpx_mock: HTTPXMock, + mcp_client: Client[FastMCPTransport], + random_module_description: Callable[[], str], + random_module_identifier: Callable[[], FuzzForgeModuleIdentifier], + random_module_name: Callable[[], str], +) -> None: + """TODO.""" + modules = [ + ModuleDefinitionSummary( + module_description=random_module_description(), + module_identifier=random_module_identifier(), + module_name=random_module_name(), + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ), + ModuleDefinitionSummary( + module_description=random_module_description(), + module_identifier=random_module_identifier(), + module_name=random_module_name(), + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ), + ] + httpx_mock.add_response( + json=GetFuzzForgeModuleDefinitionsResponse( + modules=modules, + total=2, + limit=100, + offset=0, + ).model_dump(mode="json"), + method=HTTPMethod.GET, + url="http://127.0.0.1:8000/modules", + ) + response = [ + ModuleDefinitionSummary.model_validate(entry) + for entry in loads( + cast("TextResourceContents", (await mcp_client.read_resource("fuzzforge://modules/"))[0]).text + ) + ] + assert len(response) == len(modules) + for expected, module in zip(modules, response, strict=True): + assert module.module_description == expected.module_description + assert module.module_identifier == expected.module_identifier + assert module.module_name == expected.module_name + + +async def test_get_executions_when_server_returns_valid_data( + httpx_mock: HTTPXMock, + mcp_client: Client[FastMCPTransport], + random_module_identifier: Callable[[], FuzzForgeModuleIdentifier], + random_module_execution_identifier: Callable[[], FuzzForgeExecutionIdentifier], + random_workflow_identifier: Callable[[], FuzzForgeWorkflowIdentifier], + random_workflow_execution_identifier: Callable[[], FuzzForgeExecutionIdentifier], +) -> None: + """TODO.""" + project_identifier: FuzzForgeProjectIdentifier = mcp_client.transport.server._lifespan_result.PROJECT_IDENTIFIER # type: ignore[union-attr] # noqa: SLF001 + modules = [ + ModuleExecutionSummary( + execution_identifier=random_module_execution_identifier(), + module_identifier=random_module_identifier(), + execution_status=FuzzForgeExecutionStatus.PENDING, + error=None, + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ), + ModuleExecutionSummary( + execution_identifier=random_module_execution_identifier(), + module_identifier=random_module_identifier(), + execution_status=FuzzForgeExecutionStatus.PENDING, + error=None, + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ), + ] + workflows = [ + WorkflowExecutionSummary( + execution_identifier=random_workflow_execution_identifier(), + workflow_identifier=random_workflow_identifier(), + workflow_status=FuzzForgeExecutionStatus.PENDING, + error=None, + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ), + WorkflowExecutionSummary( + execution_identifier=random_workflow_execution_identifier(), + workflow_identifier=random_workflow_identifier(), + workflow_status=FuzzForgeExecutionStatus.PENDING, + error=None, + created_at=datetime.now(tz=UTC), + updated_at=datetime.now(tz=UTC), + ), + ] + httpx_mock.add_response( + json=GetFuzzForgeModuleExecutionsResponse( + executions=modules, + project_identifier=project_identifier, + total=2, + ).model_dump(mode="json"), + method=HTTPMethod.GET, + url=f"http://127.0.0.1:8000/projects/{project_identifier}/modules", + ) + httpx_mock.add_response( + json=GetFuzzForgeWorkflowExecutionsResponse( + workflows=workflows, + project_identifier=project_identifier, + total=2, + ).model_dump(mode="json"), + method=HTTPMethod.GET, + url=f"http://127.0.0.1:8000/projects/{project_identifier}/workflows", + ) + response = loads(cast("TextResourceContents", (await mcp_client.read_resource("fuzzforge://executions/"))[0]).text) + assert len(response) == len(modules) + len(workflows) + for expected_module, module in zip( + modules, [ModuleExecutionSummary.model_validate(entry) for entry in response[: len(modules)]], strict=True + ): + assert module.execution_identifier == expected_module.execution_identifier + assert module.module_identifier == expected_module.module_identifier + for expected_workflow, workflow in zip( + workflows, [WorkflowExecutionSummary.model_validate(entry) for entry in response[len(workflows) :]], strict=True + ): + assert workflow.execution_identifier == expected_workflow.execution_identifier + assert workflow.workflow_identifier == expected_workflow.workflow_identifier diff --git a/fuzzforge-modules/cargo-fuzzer/Dockerfile b/fuzzforge-modules/cargo-fuzzer/Dockerfile new file mode 100644 index 0000000..e58bf77 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/Dockerfile @@ -0,0 +1,24 @@ +FROM localhost/fuzzforge-modules-sdk:0.1.0 + +# Install system dependencies for Rust compilation +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust toolchain with nightly (required for cargo-fuzz) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install cargo-fuzz +RUN cargo install cargo-fuzz --locked || true + +COPY ./src /app/src +COPY ./pyproject.toml /app/pyproject.toml + +# Remove workspace reference since we're using wheels +RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml + +RUN uv sync --find-links /wheels diff --git a/fuzzforge-modules/cargo-fuzzer/Makefile b/fuzzforge-modules/cargo-fuzzer/Makefile new file mode 100644 index 0000000..cada4d0 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/Makefile @@ -0,0 +1,45 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit build clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +build: + $(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION) + +save: build + $(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/cargo-fuzzer/README.md b/fuzzforge-modules/cargo-fuzzer/README.md new file mode 100644 index 0000000..d0671a1 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/README.md @@ -0,0 +1,46 @@ +# FuzzForge Modules - FIXME + +## Installation + +### Python + +```shell +# install the package (users) +uv sync +# install the package and all development dependencies (developers) +uv sync --all-extras +``` + +### Container + +```shell +# build the image +make build +# run the container +mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output" +echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json" +podman run --rm \ + --volume "${PWD}/data:/data" \ + ':' 'uv run module' +``` + +## Usage + +```shell +uv run module +``` + +## Development tools + +```shell +# run ruff (formatter) +make format +# run mypy (type checker) +make mypy +# run tests (pytest) +make pytest +# run ruff (linter) +make ruff +``` + +See the file `Makefile` at the root of this directory for more tools. diff --git a/fuzzforge-modules/cargo-fuzzer/mypy.ini b/fuzzforge-modules/cargo-fuzzer/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/cargo-fuzzer/pyproject.toml b/fuzzforge-modules/cargo-fuzzer/pyproject.toml new file mode 100644 index 0000000..904f3f8 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "cargo-fuzzer" +version = "0.1.0" +description = "FuzzForge module that runs cargo-fuzz with libFuzzer on Rust targets" +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-modules-sdk==0.0.1", + "pydantic==2.12.4", + "structlog==25.5.0", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +module = "module.__main__:main" + +[tool.uv.sources] +fuzzforge-modules-sdk = { workspace = true } + +[tool.uv] +package = true diff --git a/fuzzforge-modules/cargo-fuzzer/ruff.toml b/fuzzforge-modules/cargo-fuzzer/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/cargo-fuzzer/src/module/__init__.py b/fuzzforge-modules/cargo-fuzzer/src/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/cargo-fuzzer/src/module/__main__.py b/fuzzforge-modules/cargo-fuzzer/src/module/__main__.py new file mode 100644 index 0000000..bc8914a --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/src/module/__main__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api import logs + +from module.mod import Module + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + + +def main() -> None: + """TODO.""" + logs.configure() + module: FuzzForgeModule = Module() + module.main() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-modules/cargo-fuzzer/src/module/mod.py b/fuzzforge-modules/cargo-fuzzer/src/module/mod.py new file mode 100644 index 0000000..364a061 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/src/module/mod.py @@ -0,0 +1,516 @@ +"""Cargo Fuzzer module for FuzzForge. + +This module runs cargo-fuzz (libFuzzer) on validated Rust fuzz targets. +It takes a fuzz project with compiled harnesses and runs fuzzing for a +configurable duration, collecting crashes and statistics. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import signal +import time +from pathlib import Path +from typing import TYPE_CHECKING + +import structlog + +from fuzzforge_modules_sdk.api.constants import PATH_TO_INPUTS, PATH_TO_OUTPUTS +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults +from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + +from module.models import Input, Output, CrashInfo, FuzzingStats, TargetResult +from module.settings import Settings + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource + +logger = structlog.get_logger() + + +class Module(FuzzForgeModule): + """Cargo Fuzzer module - runs cargo-fuzz with libFuzzer on Rust targets.""" + + _settings: Settings | None + _fuzz_project_path: Path | None + _target_results: list[TargetResult] + _crashes_path: Path | None + + def __init__(self) -> None: + """Initialize an instance of the class.""" + name: str = "cargo-fuzzer" + version: str = "0.1.0" + FuzzForgeModule.__init__(self, name=name, version=version) + self._settings = None + self._fuzz_project_path = None + self._target_results = [] + self._crashes_path = None + + @classmethod + def _get_input_type(cls) -> type[Input]: + """Return the input type.""" + return Input + + @classmethod + def _get_output_type(cls) -> type[Output]: + """Return the output type.""" + return Output + + def _prepare(self, settings: Settings) -> None: # type: ignore[override] + """Prepare the module with settings. + + :param settings: Module settings. + + """ + self._settings = settings + logger.info("cargo-fuzzer preparing", settings=settings.model_dump() if settings else {}) + + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: + """Run the fuzzer. + + :param resources: Input resources (fuzz project + source). + :returns: Module execution result. + + """ + logger.info("cargo-fuzzer starting", resource_count=len(resources)) + + # Emit initial progress + self.emit_progress(0, status="initializing", message="Setting up fuzzing environment") + self.emit_event("module_started", resource_count=len(resources)) + + # Setup the fuzzing environment + if not self._setup_environment(resources): + self.emit_progress(100, status="failed", message="Failed to setup environment") + return FuzzForgeModuleResults.FAILURE + + # Get list of fuzz targets + targets = self._get_fuzz_targets() + if not targets: + logger.error("no fuzz targets found") + self.emit_progress(100, status="failed", message="No fuzz targets found") + return FuzzForgeModuleResults.FAILURE + + # Filter targets if specific ones were requested + if self._settings and self._settings.targets: + requested = set(self._settings.targets) + targets = [t for t in targets if t in requested] + if not targets: + logger.error("none of the requested targets found", requested=list(requested)) + self.emit_progress(100, status="failed", message="Requested targets not found") + return FuzzForgeModuleResults.FAILURE + + logger.info("found fuzz targets", targets=targets) + self.emit_event("targets_found", targets=targets, count=len(targets)) + + # Setup output directories + self._crashes_path = PATH_TO_OUTPUTS / "crashes" + self._crashes_path.mkdir(parents=True, exist_ok=True) + + # Run fuzzing on each target + # max_duration=0 means infinite/continuous mode + max_duration = self._settings.max_duration if self._settings else 60 + is_continuous = max_duration == 0 + + if is_continuous: + # Continuous mode: cycle through targets indefinitely + # Each target runs for 60 seconds before moving to next + duration_per_target = 60 + else: + duration_per_target = max_duration // max(len(targets), 1) + total_crashes = 0 + + # In continuous mode, loop forever; otherwise loop once + round_num = 0 + while True: + round_num += 1 + + for i, target in enumerate(targets): + if is_continuous: + progress_msg = f"Round {round_num}: Fuzzing {target}" + else: + progress_msg = f"Fuzzing target {i+1}/{len(targets)}" + + progress = int((i / len(targets)) * 100) if not is_continuous else 50 + self.emit_progress( + progress, + status="running", + message=progress_msg, + current_task=target, + metrics={ + "targets_completed": i, + "total_targets": len(targets), + "crashes_found": total_crashes, + "round": round_num if is_continuous else 1, + } + ) + self.emit_event("target_started", target=target, index=i, total=len(targets), round=round_num) + + result = self._fuzz_target(target, duration_per_target) + self._target_results.append(result) + total_crashes += len(result.crashes) + + # Emit target completion + self.emit_event( + "target_completed", + target=target, + crashes=len(result.crashes), + executions=result.stats.total_executions if result.stats else 0, + coverage=result.stats.coverage_edges if result.stats else 0, + ) + + logger.info("target completed", + target=target, + crashes=len(result.crashes), + execs=result.stats.total_executions if result.stats else 0) + + # Exit loop if not continuous mode + if not is_continuous: + break + + # Write output + self._write_output() + + # Emit final progress + self.emit_progress( + 100, + status="completed", + message=f"Fuzzing completed. Found {total_crashes} crashes.", + metrics={ + "targets_fuzzed": len(self._target_results), + "total_crashes": total_crashes, + "total_executions": sum(r.stats.total_executions for r in self._target_results if r.stats), + } + ) + self.emit_event("module_completed", total_crashes=total_crashes, targets_fuzzed=len(targets)) + + logger.info("cargo-fuzzer completed", + targets=len(self._target_results), + total_crashes=total_crashes) + + return FuzzForgeModuleResults.SUCCESS + + def _cleanup(self, settings: Settings) -> None: # type: ignore[override] + """Clean up after execution. + + :param settings: Module settings. + + """ + pass + + def _setup_environment(self, resources: list[FuzzForgeModuleResource]) -> bool: + """Setup the fuzzing environment. + + :param resources: Input resources. + :returns: True if setup successful. + + """ + import shutil + + # Find fuzz project in resources + source_fuzz_project = None + source_project_root = None + + for resource in resources: + path = Path(resource.path) + if path.is_dir(): + # Check for fuzz subdirectory + fuzz_dir = path / "fuzz" + if fuzz_dir.is_dir() and (fuzz_dir / "Cargo.toml").exists(): + source_fuzz_project = fuzz_dir + source_project_root = path + break + # Or direct fuzz project + if (path / "Cargo.toml").exists() and (path / "fuzz_targets").is_dir(): + source_fuzz_project = path + source_project_root = path.parent + break + + if source_fuzz_project is None: + logger.error("no fuzz project found in resources") + return False + + # Copy project to writable location since /data/input is read-only + # and cargo-fuzz needs to write corpus, artifacts, and build cache + work_dir = Path("/tmp/fuzz-work") + if work_dir.exists(): + shutil.rmtree(work_dir) + + # Copy the entire project root + work_project = work_dir / source_project_root.name + shutil.copytree(source_project_root, work_project, dirs_exist_ok=True) + + # Update fuzz_project_path to point to the copied location + relative_fuzz = source_fuzz_project.relative_to(source_project_root) + self._fuzz_project_path = work_project / relative_fuzz + + logger.info("using fuzz project", path=str(self._fuzz_project_path)) + return True + + def _get_fuzz_targets(self) -> list[str]: + """Get list of fuzz target names. + + :returns: List of target names. + + """ + if self._fuzz_project_path is None: + return [] + + targets = [] + fuzz_targets_dir = self._fuzz_project_path / "fuzz_targets" + + if fuzz_targets_dir.is_dir(): + for rs_file in fuzz_targets_dir.glob("*.rs"): + targets.append(rs_file.stem) + + return targets + + def _fuzz_target(self, target: str, duration: int) -> TargetResult: + """Run fuzzing on a single target. + + :param target: Name of the fuzz target. + :param duration: Maximum duration in seconds. + :returns: Fuzzing result for this target. + + """ + logger.info("fuzzing target", target=target, duration=duration) + + crashes: list[CrashInfo] = [] + stats = FuzzingStats() + + if self._fuzz_project_path is None: + return TargetResult(target=target, crashes=crashes, stats=stats) + + # Create corpus directory for this target + corpus_dir = self._fuzz_project_path / "corpus" / target + corpus_dir.mkdir(parents=True, exist_ok=True) + + # Build the command + cmd = [ + "cargo", "+nightly", "fuzz", "run", + target, + "--", + ] + + # Add time limit + if duration > 0: + cmd.append(f"-max_total_time={duration}") + + # Use fork mode to continue after crashes + # This makes libFuzzer restart worker after crash instead of exiting + cmd.append("-fork=1") + cmd.append("-ignore_crashes=1") + cmd.append("-print_final_stats=1") + + # Add jobs if specified + if self._settings and self._settings.jobs > 1: + cmd.extend([f"-jobs={self._settings.jobs}"]) + + try: + env = os.environ.copy() + env["CARGO_INCREMENTAL"] = "0" + + process = subprocess.Popen( + cmd, + cwd=self._fuzz_project_path, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + ) + + output_lines = [] + start_time = time.time() + last_metrics_emit = 0.0 + current_execs = 0 + current_cov = 0 + current_exec_s = 0 + crash_count = 0 + + # Read output with timeout (skip timeout check in infinite mode) + while True: + if process.poll() is not None: + break + + elapsed = time.time() - start_time + # Only enforce timeout if duration > 0 (not infinite mode) + if duration > 0 and elapsed > duration + 30: # Grace period + logger.warning("fuzzer timeout, terminating", target=target) + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + break + + try: + if process.stdout: + line = process.stdout.readline() + if line: + output_lines.append(line) + + # Parse real-time metrics from libFuzzer output + # Example: "#12345 NEW cov: 100 ft: 50 corp: 25/1Kb exec/s: 1000" + exec_match = re.search(r"#(\d+)", line) + if exec_match: + current_execs = int(exec_match.group(1)) + + cov_match = re.search(r"cov:\s*(\d+)", line) + if cov_match: + current_cov = int(cov_match.group(1)) + + exec_s_match = re.search(r"exec/s:\s*(\d+)", line) + if exec_s_match: + current_exec_s = int(exec_s_match.group(1)) + + # Check for crash indicators + if "SUMMARY:" in line or "ERROR:" in line or "crash-" in line.lower(): + crash_count += 1 + self.emit_event( + "crash_detected", + target=target, + crash_number=crash_count, + line=line.strip(), + ) + logger.debug("fuzzer output", line=line.strip()) + + # Emit metrics periodically (every 2 seconds) + if elapsed - last_metrics_emit >= 2.0: + last_metrics_emit = elapsed + self.emit_event( + "metrics", + target=target, + executions=current_execs, + coverage=current_cov, + exec_per_sec=current_exec_s, + crashes=crash_count, + elapsed_seconds=int(elapsed), + remaining_seconds=max(0, duration - int(elapsed)), + ) + + except Exception: + pass + + # Parse statistics from output + stats = self._parse_fuzzer_stats(output_lines) + + # Collect crashes + crashes = self._collect_crashes(target) + + # Emit final event for this target if crashes were found + if crashes: + self.emit_event( + "crashes_collected", + target=target, + count=len(crashes), + paths=[c.file_path for c in crashes], + ) + + except FileNotFoundError: + logger.error("cargo-fuzz not found, please install with: cargo install cargo-fuzz") + stats.error = "cargo-fuzz not installed" + self.emit_event("error", target=target, message="cargo-fuzz not installed") + except Exception as e: + logger.exception("fuzzing error", target=target, error=str(e)) + stats.error = str(e) + self.emit_event("error", target=target, message=str(e)) + + return TargetResult(target=target, crashes=crashes, stats=stats) + + def _parse_fuzzer_stats(self, output_lines: list[str]) -> FuzzingStats: + """Parse fuzzer output for statistics. + + :param output_lines: Lines of fuzzer output. + :returns: Parsed statistics. + + """ + stats = FuzzingStats() + full_output = "".join(output_lines) + + # Parse libFuzzer stats + # Example: "#12345 DONE cov: 100 ft: 50 corp: 25/1Kb exec/s: 1000" + exec_match = re.search(r"#(\d+)", full_output) + if exec_match: + stats.total_executions = int(exec_match.group(1)) + + cov_match = re.search(r"cov:\s*(\d+)", full_output) + if cov_match: + stats.coverage_edges = int(cov_match.group(1)) + + corp_match = re.search(r"corp:\s*(\d+)", full_output) + if corp_match: + stats.corpus_size = int(corp_match.group(1)) + + exec_s_match = re.search(r"exec/s:\s*(\d+)", full_output) + if exec_s_match: + stats.executions_per_second = int(exec_s_match.group(1)) + + return stats + + def _collect_crashes(self, target: str) -> list[CrashInfo]: + """Collect crash files from fuzzer output. + + :param target: Name of the fuzz target. + :returns: List of crash info. + + """ + crashes: list[CrashInfo] = [] + + if self._fuzz_project_path is None or self._crashes_path is None: + return crashes + + # Check for crashes in the artifacts directory + artifacts_dir = self._fuzz_project_path / "artifacts" / target + + if artifacts_dir.is_dir(): + for crash_file in artifacts_dir.glob("crash-*"): + if crash_file.is_file(): + # Copy crash to output + output_crash = self._crashes_path / target + output_crash.mkdir(parents=True, exist_ok=True) + dest = output_crash / crash_file.name + shutil.copy2(crash_file, dest) + + # Read crash input + crash_data = crash_file.read_bytes() + + crash_info = CrashInfo( + file_path=str(dest), + input_hash=crash_file.name, + input_size=len(crash_data), + ) + crashes.append(crash_info) + + logger.info("found crash", target=target, file=crash_file.name) + + return crashes + + def _write_output(self) -> None: + """Write the fuzzing results to output.""" + output_path = PATH_TO_OUTPUTS / "fuzzing_results.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + + total_crashes = sum(len(r.crashes) for r in self._target_results) + total_execs = sum(r.stats.total_executions for r in self._target_results if r.stats) + + output_data = { + "fuzz_project": str(self._fuzz_project_path), + "targets_fuzzed": len(self._target_results), + "total_crashes": total_crashes, + "total_executions": total_execs, + "crashes_path": str(self._crashes_path), + "results": [ + { + "target": r.target, + "crashes": [c.model_dump() for c in r.crashes], + "stats": r.stats.model_dump() if r.stats else None, + } + for r in self._target_results + ], + } + + output_path.write_text(json.dumps(output_data, indent=2)) + logger.info("wrote fuzzing results", path=str(output_path)) diff --git a/fuzzforge-modules/cargo-fuzzer/src/module/models.py b/fuzzforge-modules/cargo-fuzzer/src/module/models.py new file mode 100644 index 0000000..9c4fb9e --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/src/module/models.py @@ -0,0 +1,88 @@ +"""Models for the cargo-fuzzer module.""" + +from pydantic import BaseModel, Field +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase + +from module.settings import Settings + + +class FuzzingStats(BaseModel): + """Statistics from a fuzzing run.""" + + #: Total number of test case executions + total_executions: int = 0 + + #: Executions per second + executions_per_second: int = 0 + + #: Number of coverage edges discovered + coverage_edges: int = 0 + + #: Size of the corpus + corpus_size: int = 0 + + #: Any error message + error: str = "" + + +class CrashInfo(BaseModel): + """Information about a discovered crash.""" + + #: Path to the crash input file + file_path: str + + #: Hash/name of the crash input + input_hash: str + + #: Size of the crash input in bytes + input_size: int = 0 + + #: Crash type (if identified) + crash_type: str = "" + + #: Stack trace (if available) + stack_trace: str = "" + + +class TargetResult(BaseModel): + """Result of fuzzing a single target.""" + + #: Name of the fuzz target + target: str + + #: List of crashes found + crashes: list[CrashInfo] = Field(default_factory=list) + + #: Fuzzing statistics + stats: FuzzingStats = Field(default_factory=FuzzingStats) + + +class Input(FuzzForgeModuleInputBase[Settings]): + """Input for the cargo-fuzzer module. + + Expects: + - A fuzz project directory with validated harnesses + - Optionally the source crate to link against + """ + + +class Output(FuzzForgeModuleOutputBase): + """Output from the cargo-fuzzer module.""" + + #: Path to the fuzz project + fuzz_project: str = "" + + #: Number of targets fuzzed + targets_fuzzed: int = 0 + + #: Total crashes found across all targets + total_crashes: int = 0 + + #: Total executions across all targets + total_executions: int = 0 + + #: Path to collected crash files + crashes_path: str = "" + + #: Results per target + results: list[TargetResult] = Field(default_factory=list) diff --git a/fuzzforge-modules/cargo-fuzzer/src/module/settings.py b/fuzzforge-modules/cargo-fuzzer/src/module/settings.py new file mode 100644 index 0000000..ec49c21 --- /dev/null +++ b/fuzzforge-modules/cargo-fuzzer/src/module/settings.py @@ -0,0 +1,35 @@ +"""Settings for the cargo-fuzzer module.""" + +from typing import Optional +from pydantic import model_validator +from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase + + +class Settings(FuzzForgeModulesSettingsBase): + """Settings for the cargo-fuzzer module.""" + + #: Maximum fuzzing duration in seconds (total across all targets) + #: Set to 0 for infinite/continuous mode + max_duration: int = 60 + + #: Number of parallel fuzzing jobs + jobs: int = 1 + + #: Maximum length of generated inputs + max_len: int = 4096 + + #: Whether to use AddressSanitizer + use_asan: bool = True + + #: Specific targets to fuzz (empty = all targets) + targets: list[str] = [] + + #: Single target to fuzz (convenience alias for targets) + target: Optional[str] = None + + @model_validator(mode="after") + def handle_single_target(self) -> "Settings": + """Convert single target to targets list if provided.""" + if self.target and self.target not in self.targets: + self.targets.append(self.target) + return self diff --git a/fuzzforge-modules/cargo-fuzzer/tests/.gitkeep b/fuzzforge-modules/cargo-fuzzer/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/crash-analyzer/Dockerfile b/fuzzforge-modules/crash-analyzer/Dockerfile new file mode 100644 index 0000000..8343893 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/Dockerfile @@ -0,0 +1,9 @@ +FROM localhost/fuzzforge-modules-sdk:0.1.0 + +COPY ./src /app/src +COPY ./pyproject.toml /app/pyproject.toml + +# Remove workspace reference since we're using wheels +RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml + +RUN uv sync --find-links /wheels diff --git a/fuzzforge-modules/crash-analyzer/Makefile b/fuzzforge-modules/crash-analyzer/Makefile new file mode 100644 index 0000000..cada4d0 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/Makefile @@ -0,0 +1,45 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit build clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +build: + $(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION) + +save: build + $(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/crash-analyzer/README.md b/fuzzforge-modules/crash-analyzer/README.md new file mode 100644 index 0000000..d0671a1 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/README.md @@ -0,0 +1,46 @@ +# FuzzForge Modules - FIXME + +## Installation + +### Python + +```shell +# install the package (users) +uv sync +# install the package and all development dependencies (developers) +uv sync --all-extras +``` + +### Container + +```shell +# build the image +make build +# run the container +mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output" +echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json" +podman run --rm \ + --volume "${PWD}/data:/data" \ + ':' 'uv run module' +``` + +## Usage + +```shell +uv run module +``` + +## Development tools + +```shell +# run ruff (formatter) +make format +# run mypy (type checker) +make mypy +# run tests (pytest) +make pytest +# run ruff (linter) +make ruff +``` + +See the file `Makefile` at the root of this directory for more tools. diff --git a/fuzzforge-modules/crash-analyzer/mypy.ini b/fuzzforge-modules/crash-analyzer/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/crash-analyzer/pyproject.toml b/fuzzforge-modules/crash-analyzer/pyproject.toml new file mode 100644 index 0000000..6ee9926 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "crash-analyzer" +version = "0.1.0" +description = "FuzzForge module that analyzes fuzzing crashes and generates security reports" +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-modules-sdk==0.0.1", + "pydantic==2.12.4", + "structlog==25.5.0", + "jinja2==3.1.6", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +module = "module.__main__:main" + +[tool.uv.sources] +fuzzforge-modules-sdk = { workspace = true } + +[tool.uv] +package = true diff --git a/fuzzforge-modules/crash-analyzer/ruff.toml b/fuzzforge-modules/crash-analyzer/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/crash-analyzer/src/module/__init__.py b/fuzzforge-modules/crash-analyzer/src/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/crash-analyzer/src/module/__main__.py b/fuzzforge-modules/crash-analyzer/src/module/__main__.py new file mode 100644 index 0000000..bc8914a --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/src/module/__main__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api import logs + +from module.mod import Module + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + + +def main() -> None: + """TODO.""" + logs.configure() + module: FuzzForgeModule = Module() + module.main() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-modules/crash-analyzer/src/module/mod.py b/fuzzforge-modules/crash-analyzer/src/module/mod.py new file mode 100644 index 0000000..40f71ff --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/src/module/mod.py @@ -0,0 +1,340 @@ +"""Crash Analyzer module for FuzzForge. + +This module analyzes crashes from cargo-fuzz, deduplicates them, +extracts stack traces, and triages them by severity. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +import structlog + +from fuzzforge_modules_sdk.api.constants import PATH_TO_INPUTS, PATH_TO_OUTPUTS +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults +from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + +from module.models import Input, Output, CrashAnalysis, Severity +from module.settings import Settings + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource + +logger = structlog.get_logger() + + +class Module(FuzzForgeModule): + """Crash Analyzer module - analyzes and triages fuzzer crashes.""" + + _settings: Settings | None + _analyses: list[CrashAnalysis] + _fuzz_project_path: Path | None + + def __init__(self) -> None: + """Initialize an instance of the class.""" + name: str = "crash-analyzer" + version: str = "0.1.0" + FuzzForgeModule.__init__(self, name=name, version=version) + self._settings = None + self._analyses = [] + self._fuzz_project_path = None + + @classmethod + def _get_input_type(cls) -> type[Input]: + """Return the input type.""" + return Input + + @classmethod + def _get_output_type(cls) -> type[Output]: + """Return the output type.""" + return Output + + def _prepare(self, settings: Settings) -> None: # type: ignore[override] + """Prepare the module. + + :param settings: Module settings. + + """ + self._settings = settings + logger.info("crash-analyzer preparing", settings=settings.model_dump() if settings else {}) + + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: + """Run the crash analyzer. + + :param resources: Input resources (fuzzing results + crashes). + :returns: Module execution result. + + """ + logger.info("crash-analyzer starting", resource_count=len(resources)) + + # Find crashes directory and fuzz project + crashes_path = None + for resource in resources: + path = Path(resource.path) + if path.is_dir(): + if path.name == "crashes" or (path / "crashes").is_dir(): + crashes_path = path if path.name == "crashes" else path / "crashes" + if (path / "fuzz_targets").is_dir(): + self._fuzz_project_path = path + if (path / "fuzz" / "fuzz_targets").is_dir(): + self._fuzz_project_path = path / "fuzz" + + if crashes_path is None: + # Try to find crashes in fuzzing_results.json + for resource in resources: + path = Path(resource.path) + if path.name == "fuzzing_results.json" and path.exists(): + with open(path) as f: + data = json.load(f) + if "crashes_path" in data: + crashes_path = Path(data["crashes_path"]) + break + + if crashes_path is None or not crashes_path.exists(): + logger.warning("no crashes found to analyze") + self._write_output() + return FuzzForgeModuleResults.SUCCESS + + logger.info("analyzing crashes", path=str(crashes_path)) + + # Analyze crashes per target + for target_dir in crashes_path.iterdir(): + if target_dir.is_dir(): + target = target_dir.name + for crash_file in target_dir.glob("crash-*"): + if crash_file.is_file(): + analysis = self._analyze_crash(target, crash_file) + self._analyses.append(analysis) + + # Deduplicate crashes + self._deduplicate_crashes() + + # Write output + self._write_output() + + unique_count = sum(1 for a in self._analyses if not a.is_duplicate) + logger.info("crash-analyzer completed", + total=len(self._analyses), + unique=unique_count) + + return FuzzForgeModuleResults.SUCCESS + + def _cleanup(self, settings: Settings) -> None: # type: ignore[override] + """Clean up after execution. + + :param settings: Module settings. + + """ + pass + + def _analyze_crash(self, target: str, crash_file: Path) -> CrashAnalysis: + """Analyze a single crash. + + :param target: Name of the fuzz target. + :param crash_file: Path to the crash input file. + :returns: Crash analysis result. + + """ + logger.debug("analyzing crash", target=target, file=crash_file.name) + + # Read crash input + crash_data = crash_file.read_bytes() + input_hash = hashlib.sha256(crash_data).hexdigest()[:16] + + # Try to reproduce and get stack trace + stack_trace = "" + crash_type = "unknown" + severity = Severity.UNKNOWN + + if self._fuzz_project_path: + stack_trace, crash_type = self._reproduce_crash(target, crash_file) + severity = self._determine_severity(crash_type, stack_trace) + + return CrashAnalysis( + target=target, + input_file=str(crash_file), + input_hash=input_hash, + input_size=len(crash_data), + crash_type=crash_type, + severity=severity, + stack_trace=stack_trace, + is_duplicate=False, + ) + + def _reproduce_crash(self, target: str, crash_file: Path) -> tuple[str, str]: + """Reproduce a crash to get stack trace. + + :param target: Name of the fuzz target. + :param crash_file: Path to the crash input file. + :returns: Tuple of (stack_trace, crash_type). + + """ + if self._fuzz_project_path is None: + return "", "unknown" + + try: + env = os.environ.copy() + env["RUST_BACKTRACE"] = "1" + + result = subprocess.run( + [ + "cargo", "+nightly", "fuzz", "run", + target, + str(crash_file), + "--", + "-runs=1", + ], + cwd=self._fuzz_project_path, + capture_output=True, + text=True, + timeout=30, + env=env, + ) + + output = result.stdout + result.stderr + + # Extract crash type + crash_type = "unknown" + if "heap-buffer-overflow" in output.lower(): + crash_type = "heap-buffer-overflow" + elif "stack-buffer-overflow" in output.lower(): + crash_type = "stack-buffer-overflow" + elif "heap-use-after-free" in output.lower(): + crash_type = "use-after-free" + elif "null" in output.lower() and "deref" in output.lower(): + crash_type = "null-pointer-dereference" + elif "panic" in output.lower(): + crash_type = "panic" + elif "assertion" in output.lower(): + crash_type = "assertion-failure" + elif "timeout" in output.lower(): + crash_type = "timeout" + elif "out of memory" in output.lower() or "oom" in output.lower(): + crash_type = "out-of-memory" + + # Extract stack trace + stack_lines = [] + in_stack = False + for line in output.splitlines(): + if "SUMMARY:" in line or "ERROR:" in line: + in_stack = True + if in_stack: + stack_lines.append(line) + if len(stack_lines) > 50: # Limit stack trace length + break + + return "\n".join(stack_lines), crash_type + + except subprocess.TimeoutExpired: + return "", "timeout" + except Exception as e: + logger.warning("failed to reproduce crash", error=str(e)) + return "", "unknown" + + def _determine_severity(self, crash_type: str, stack_trace: str) -> Severity: + """Determine crash severity based on type and stack trace. + + :param crash_type: Type of the crash. + :param stack_trace: Stack trace string. + :returns: Severity level. + + """ + high_severity = [ + "heap-buffer-overflow", + "stack-buffer-overflow", + "use-after-free", + "double-free", + ] + + medium_severity = [ + "null-pointer-dereference", + "out-of-memory", + "integer-overflow", + ] + + low_severity = [ + "panic", + "assertion-failure", + "timeout", + ] + + if crash_type in high_severity: + return Severity.HIGH + elif crash_type in medium_severity: + return Severity.MEDIUM + elif crash_type in low_severity: + return Severity.LOW + else: + return Severity.UNKNOWN + + def _deduplicate_crashes(self) -> None: + """Mark duplicate crashes based on stack trace similarity.""" + seen_signatures: set[str] = set() + + for analysis in self._analyses: + # Create a signature from crash type and key stack frames + signature = self._create_signature(analysis) + + if signature in seen_signatures: + analysis.is_duplicate = True + else: + seen_signatures.add(signature) + + def _create_signature(self, analysis: CrashAnalysis) -> str: + """Create a unique signature for a crash. + + :param analysis: Crash analysis. + :returns: Signature string. + + """ + # Use crash type + first few significant stack frames + parts = [analysis.target, analysis.crash_type] + + # Extract function names from stack trace + func_pattern = re.compile(r"in (\S+)") + funcs = func_pattern.findall(analysis.stack_trace) + + # Use first 3 unique functions + seen = set() + for func in funcs: + if func not in seen and not func.startswith("std::"): + parts.append(func) + seen.add(func) + if len(seen) >= 3: + break + + return "|".join(parts) + + def _write_output(self) -> None: + """Write the analysis results to output.""" + output_path = PATH_TO_OUTPUTS / "crash_analysis.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + + unique = [a for a in self._analyses if not a.is_duplicate] + duplicates = [a for a in self._analyses if a.is_duplicate] + + # Group by severity + by_severity = { + "high": [a for a in unique if a.severity == Severity.HIGH], + "medium": [a for a in unique if a.severity == Severity.MEDIUM], + "low": [a for a in unique if a.severity == Severity.LOW], + "unknown": [a for a in unique if a.severity == Severity.UNKNOWN], + } + + output_data = { + "total_crashes": len(self._analyses), + "unique_crashes": len(unique), + "duplicate_crashes": len(duplicates), + "severity_summary": {k: len(v) for k, v in by_severity.items()}, + "unique_analyses": [a.model_dump() for a in unique], + "duplicate_analyses": [a.model_dump() for a in duplicates], + } + + output_path.write_text(json.dumps(output_data, indent=2, default=str)) + logger.info("wrote crash analysis", path=str(output_path)) diff --git a/fuzzforge-modules/crash-analyzer/src/module/models.py b/fuzzforge-modules/crash-analyzer/src/module/models.py new file mode 100644 index 0000000..bf8620c --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/src/module/models.py @@ -0,0 +1,79 @@ +"""Models for the crash-analyzer module.""" + +from enum import Enum + +from pydantic import BaseModel, Field +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase + +from module.settings import Settings + + +class Severity(str, Enum): + """Severity level of a crash.""" + + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + UNKNOWN = "unknown" + + +class CrashAnalysis(BaseModel): + """Analysis of a single crash.""" + + #: Name of the fuzz target + target: str + + #: Path to the input file that caused the crash + input_file: str + + #: Hash of the input for identification + input_hash: str + + #: Size of the input in bytes + input_size: int = 0 + + #: Type of crash (e.g., "heap-buffer-overflow", "panic") + crash_type: str = "unknown" + + #: Severity level + severity: Severity = Severity.UNKNOWN + + #: Stack trace from reproducing the crash + stack_trace: str = "" + + #: Whether this crash is a duplicate of another + is_duplicate: bool = False + + #: Signature for deduplication + signature: str = "" + + +class Input(FuzzForgeModuleInputBase[Settings]): + """Input for the crash-analyzer module. + + Expects: + - Crashes directory from cargo-fuzzer + - Optionally the fuzz project for reproduction + """ + + +class Output(FuzzForgeModuleOutputBase): + """Output from the crash-analyzer module.""" + + #: Total number of crashes analyzed + total_crashes: int = 0 + + #: Number of unique crashes (after deduplication) + unique_crashes: int = 0 + + #: Number of duplicate crashes + duplicate_crashes: int = 0 + + #: Summary by severity + severity_summary: dict[str, int] = Field(default_factory=dict) + + #: Unique crash analyses + unique_analyses: list[CrashAnalysis] = Field(default_factory=list) + + #: Duplicate crash analyses + duplicate_analyses: list[CrashAnalysis] = Field(default_factory=list) diff --git a/fuzzforge-modules/crash-analyzer/src/module/settings.py b/fuzzforge-modules/crash-analyzer/src/module/settings.py new file mode 100644 index 0000000..fdfaf62 --- /dev/null +++ b/fuzzforge-modules/crash-analyzer/src/module/settings.py @@ -0,0 +1,16 @@ +"""Settings for the crash-analyzer module.""" + +from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase + + +class Settings(FuzzForgeModulesSettingsBase): + """Settings for the crash-analyzer module.""" + + #: Whether to reproduce crashes for stack traces + reproduce_crashes: bool = True + + #: Timeout for reproducing each crash (seconds) + reproduce_timeout: int = 30 + + #: Whether to deduplicate crashes + deduplicate: bool = True diff --git a/fuzzforge-modules/crash-analyzer/tests/.gitkeep b/fuzzforge-modules/crash-analyzer/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-module-template/Dockerfile b/fuzzforge-modules/fuzzforge-module-template/Dockerfile new file mode 100644 index 0000000..222f061 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/Dockerfile @@ -0,0 +1,9 @@ +FROM localhost/fuzzforge-modules-sdk:0.0.1 + +COPY ./src /app/src +COPY ./pyproject.toml /app/pyproject.toml + +# Remove workspace reference since we're using wheels +RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml + +RUN uv sync --find-links /wheels diff --git a/fuzzforge-modules/fuzzforge-module-template/Makefile b/fuzzforge-modules/fuzzforge-module-template/Makefile new file mode 100644 index 0000000..cada4d0 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/Makefile @@ -0,0 +1,45 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit build clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +build: + $(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION) + +save: build + $(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/fuzzforge-module-template/README.md b/fuzzforge-modules/fuzzforge-module-template/README.md new file mode 100644 index 0000000..d0671a1 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/README.md @@ -0,0 +1,46 @@ +# FuzzForge Modules - FIXME + +## Installation + +### Python + +```shell +# install the package (users) +uv sync +# install the package and all development dependencies (developers) +uv sync --all-extras +``` + +### Container + +```shell +# build the image +make build +# run the container +mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output" +echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json" +podman run --rm \ + --volume "${PWD}/data:/data" \ + ':' 'uv run module' +``` + +## Usage + +```shell +uv run module +``` + +## Development tools + +```shell +# run ruff (formatter) +make format +# run mypy (type checker) +make mypy +# run tests (pytest) +make pytest +# run ruff (linter) +make ruff +``` + +See the file `Makefile` at the root of this directory for more tools. diff --git a/fuzzforge-modules/fuzzforge-module-template/mypy.ini b/fuzzforge-modules/fuzzforge-module-template/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/fuzzforge-module-template/pyproject.toml b/fuzzforge-modules/fuzzforge-module-template/pyproject.toml new file mode 100644 index 0000000..29dea4b --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "fuzzforge-module-template" +version = "0.0.1" +description = "FIXME" +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-modules-sdk==0.0.1", + "pydantic==2.12.4", + "structlog==25.5.0", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +module = "module.__main__:main" + +[tool.uv.sources] +fuzzforge-modules-sdk = { workspace = true } + +[tool.uv] +package = true diff --git a/fuzzforge-modules/fuzzforge-module-template/ruff.toml b/fuzzforge-modules/fuzzforge-module-template/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/fuzzforge-module-template/src/module/__init__.py b/fuzzforge-modules/fuzzforge-module-template/src/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-module-template/src/module/__main__.py b/fuzzforge-modules/fuzzforge-module-template/src/module/__main__.py new file mode 100644 index 0000000..bc8914a --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/src/module/__main__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api import logs + +from module.mod import Module + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + + +def main() -> None: + """TODO.""" + logs.configure() + module: FuzzForgeModule = Module() + module.main() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-modules/fuzzforge-module-template/src/module/mod.py b/fuzzforge-modules/fuzzforge-module-template/src/module/mod.py new file mode 100644 index 0000000..f0f85e9 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/src/module/mod.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults +from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + +from module.models import Input, Output + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource, FuzzForgeModulesSettingsType + + +class Module(FuzzForgeModule): + """TODO.""" + + def __init__(self) -> None: + """Initialize an instance of the class.""" + name: str = "FIXME" + version: str = "FIXME" + FuzzForgeModule.__init__(self, name=name, version=version) + + @classmethod + def _get_input_type(cls) -> type[Input]: + """TODO.""" + return Input + + @classmethod + def _get_output_type(cls) -> type[Output]: + """TODO.""" + return Output + + def _prepare(self, settings: FuzzForgeModulesSettingsType) -> None: + """TODO. + + :param settings: TODO. + + """ + + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: # noqa: ARG002 + """TODO. + + :param resources: TODO. + :returns: TODO. + + """ + return FuzzForgeModuleResults.SUCCESS + + def _cleanup(self, settings: FuzzForgeModulesSettingsType) -> None: + """TODO. + + :param settings: TODO. + + """ diff --git a/fuzzforge-modules/fuzzforge-module-template/src/module/models.py b/fuzzforge-modules/fuzzforge-module-template/src/module/models.py new file mode 100644 index 0000000..2a3f021 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/src/module/models.py @@ -0,0 +1,11 @@ +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase + +from module.settings import Settings + + +class Input(FuzzForgeModuleInputBase[Settings]): + """TODO.""" + + +class Output(FuzzForgeModuleOutputBase): + """TODO.""" diff --git a/fuzzforge-modules/fuzzforge-module-template/src/module/settings.py b/fuzzforge-modules/fuzzforge-module-template/src/module/settings.py new file mode 100644 index 0000000..f916ad4 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-module-template/src/module/settings.py @@ -0,0 +1,7 @@ +from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase + + +class Settings(FuzzForgeModulesSettingsBase): + """TODO.""" + + # Here goes your attributes diff --git a/fuzzforge-modules/fuzzforge-module-template/tests/.gitkeep b/fuzzforge-modules/fuzzforge-module-template/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/Dockerfile b/fuzzforge-modules/fuzzforge-modules-sdk/Dockerfile new file mode 100644 index 0000000..c98782a --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/Dockerfile @@ -0,0 +1,30 @@ +# FuzzForge Modules SDK - Base image for all modules +# +# This image provides: +# - Python 3.14 with uv package manager +# - Pre-built wheels for common dependencies +# - Standard module directory structure + +FROM ghcr.io/astral-sh/uv:python3.14-bookworm-slim + +# Install system dependencies commonly needed by modules +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Set up application directory structure +WORKDIR /app + +# Create FuzzForge standard directories +RUN mkdir -p /fuzzforge/input /fuzzforge/output + +# Copy wheels directory (built by parent Makefile) +COPY .wheels /wheels + +# Set up uv for the container +ENV UV_SYSTEM_PYTHON=1 +ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy + +# Default entrypoint - modules override this +ENTRYPOINT ["uv", "run", "module"] diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/Makefile b/fuzzforge-modules/fuzzforge-modules-sdk/Makefile new file mode 100644 index 0000000..e7ce0a9 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/Makefile @@ -0,0 +1,39 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +SOURCES=./src +TESTS=./tests + +FUZZFORGE_MODULE_TEMPLATE=$(PWD)/src/fuzzforge_modules_sdk/templates/module + +.PHONY: bandit clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/README.md b/fuzzforge-modules/fuzzforge-modules-sdk/README.md new file mode 100644 index 0000000..334325b --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/README.md @@ -0,0 +1,67 @@ +# FuzzForge Modules SDK + +... + +# Setup + +- start the podman user socket + +```shell +systemctl --user start podman.socket +``` + +NB : you can also automaticllay start it at boot + +```shell +systemctl --user enable --now podman.socket +``` + +## HACK : fix missing `fuzzforge-modules-sdk` + +- if you have this error when using some fuzzforge-modules-sdk deps : + +```shell +❯ make format +uv run ruff format ./src ./tests + × No solution found when resolving dependencies: + ╰─▶ Because fuzzforge-modules-sdk was not found in the package registry and your project depends on fuzzforge-modules-sdk==0.0.1, we can + conclude that your project's requirements are unsatisfiable. + And because your project requires opengrep[lints], we can conclude that your project's requirements are unsatisfiable. +make: *** [Makefile:30: format] Error 1 +``` + +- build a wheel package of fuzzforge-modules-sdk + +```shell +cd fuzzforge_ng/fuzzforge-modules/fuzzforge-modules-sdk +uv build +``` + +- then inside your module project, install it + +```shell +cd fuzzforge_ng_modules/mymodule +uv sync --all-extras --find-links ../../fuzzforge_ng/dist/ +``` + +# Usage + +## Prepare + +- enter venv (or use uv run) + +```shell +source .venv/bin/activate +``` + +- create a new module + +```shell +fuzzforge-modules-sdk new module --name my_new_module --directory ../fuzzforge_ng_modules/ +``` + +- build the base image + +```shell +fuzzforge-modules-sdk build image +``` \ No newline at end of file diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/mypy.ini b/fuzzforge-modules/fuzzforge-modules-sdk/mypy.ini new file mode 100644 index 0000000..f74350d --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +exclude = ^src/fuzzforge_modules_sdk/templates/.* +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/pyproject.toml b/fuzzforge-modules/fuzzforge-modules-sdk/pyproject.toml new file mode 100644 index 0000000..b330d94 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "fuzzforge-modules-sdk" +version = "0.0.1" +description = "Software development kit (SDK) for FuzzForge's modules." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "podman==5.6.0", + "pydantic==2.12.4", + "tomlkit==0.13.3", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +fuzzforge-modules-sdk = "fuzzforge_modules_sdk._cli.main:main" + +[tool.setuptools.package-data] +fuzzforge_modules_sdk = [ + "assets/**/*", + "templates/**/*", +] diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/ruff.toml b/fuzzforge-modules/fuzzforge-modules-sdk/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/__init__.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/__init__.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/build_base_image.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/build_base_image.py new file mode 100644 index 0000000..3cc04cb --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/build_base_image.py @@ -0,0 +1,66 @@ +from importlib.resources import files +from pathlib import Path +from shutil import copyfile, copytree +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING, Literal + +import os + +from podman import PodmanClient +from tomlkit import TOMLDocument, parse + +if TYPE_CHECKING: + from importlib.resources.abc import Traversable + + +def _get_default_podman_socket() -> str: + """Get the default Podman socket path for the current user.""" + uid = os.getuid() + return f"unix:///run/user/{uid}/podman/podman.sock" + + +PATH_TO_SOURCES: Path = Path(__file__).parent.parent + + +def _build_podman_image(directory: Path, tag: str, socket: str | None = None) -> None: + if socket is None: + socket = _get_default_podman_socket() + with PodmanClient(base_url=socket) as client: + client.images.build( + dockerfile="Dockerfile", + nocache=True, + path=directory, + tag=tag, + ) + + +def build_base_image(engine: Literal["podman"], socket: str | None = None) -> None: + with TemporaryDirectory() as directory: + path_to_assets: Traversable = files("fuzzforge_modules_sdk").joinpath("assets") + copyfile( + src=str(path_to_assets.joinpath("Dockerfile")), + dst=Path(directory).joinpath("Dockerfile"), + ) + copyfile( + src=str(path_to_assets.joinpath("pyproject.toml")), + dst=Path(directory).joinpath("pyproject.toml"), + ) + copytree(src=str(PATH_TO_SOURCES), dst=Path(directory).joinpath("src").joinpath(PATH_TO_SOURCES.name)) + + # update the file 'pyproject.toml' + path: Path = Path(directory).joinpath("pyproject.toml") + data: TOMLDocument = parse(path.read_text()) + name: str = data["project"]["name"] # type: ignore[assignment, index] + version: str = data["project"]["version"] # type: ignore[assignment, index] + tag: str = f"{name}:{version}" + + match engine: + case "podman": + _build_podman_image( + directory=Path(directory), + socket=socket, + tag=tag, + ) + case _: + message: str = f"unsupported engine '{engine}'" + raise Exception(message) # noqa: TRY002 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/create_new_module.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/create_new_module.py new file mode 100644 index 0000000..0c2001c --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/create_new_module.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from importlib.resources import files +from shutil import copytree, ignore_patterns +from typing import TYPE_CHECKING + +from tomlkit import dumps, parse + +if TYPE_CHECKING: + from importlib.resources.abc import Traversable + from pathlib import Path + + from tomlkit import TOMLDocument + + +def create_new_module(name: str, directory: Path) -> None: + source: Traversable = files("fuzzforge_modules_sdk").joinpath("templates").joinpath("fuzzforge-module-template") + destination: Path = directory.joinpath(name) # TODO: sanitize path + copytree( + src=str(source), + dst=destination, + ignore=ignore_patterns("__pycache__", "*.egg-info", "*.pyc", ".mypy_cache", ".ruff_cache", ".venv"), + ) + + # update the file 'pyproject.toml' + path: Path = destination.joinpath("pyproject.toml") + data: TOMLDocument = parse(path.read_text()) + data["project"]["name"] = name # type: ignore[index] + del data["tool"]["uv"]["sources"] # type: ignore[index, union-attr] + path.write_text(dumps(data)) diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/main.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/main.py new file mode 100644 index 0000000..e6ab418 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/_cli/main.py @@ -0,0 +1,71 @@ +from argparse import ArgumentParser +from pathlib import Path + +from fuzzforge_modules_sdk._cli.build_base_image import build_base_image +from fuzzforge_modules_sdk._cli.create_new_module import create_new_module + + +def create_parser() -> ArgumentParser: + parser: ArgumentParser = ArgumentParser( + prog="fuzzforge-modules-sdk", description="Utilities for the Fuzzforge Modules SDK." + ) + + subparsers = parser.add_subparsers(required=True) + + # fuzzforge-modules-sdk build ... + parser_build = subparsers.add_parser(name="build") + + subparsers_build = parser_build.add_subparsers(required=True) + + # fuzzforge-modules-sdk build image ... + parser_build_image = subparsers_build.add_parser( + name="image", + help="Build the image.", + ) + parser_build_image.add_argument( + "--engine", + default="podman", + ) + parser_build_image.add_argument( + "--socket", + default=None, + ) + parser_build_image.set_defaults( + function_to_execute=build_base_image, + ) + + # fuzzforge-modules-sdk new ... + parser_new = subparsers.add_parser(name="new") + + subparsers_new = parser_new.add_subparsers(required=True) + + # fuzzforge-modules-sdk new module ... + parser_new_module = subparsers_new.add_parser( + name="module", + help="Generate the boilerplate required to create a new module.", + ) + parser_new_module.add_argument( + "--name", + help="The name of the module to create.", + required=True, + ) + parser_new_module.add_argument( + "--directory", + default=".", + type=Path, + help="The directory the new module should be created into (defaults to current working directory).", + ) + parser_new_module.set_defaults( + function_to_execute=create_new_module, + ) + + return parser + + +def main() -> None: + """Entry point for the command-line interface.""" + parser: ArgumentParser = create_parser() + arguments = parser.parse_args() + function_to_execute = arguments.function_to_execute + del arguments.function_to_execute + function_to_execute(**vars(arguments)) diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/__init__.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/constants.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/constants.py new file mode 100644 index 0000000..47f6a35 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/constants.py @@ -0,0 +1,13 @@ +from pathlib import Path + +PATH_TO_DATA: Path = Path("/data") +PATH_TO_INPUTS: Path = PATH_TO_DATA.joinpath("input") +PATH_TO_INPUT: Path = PATH_TO_INPUTS.joinpath("input.json") +PATH_TO_OUTPUTS: Path = PATH_TO_DATA.joinpath("output") +PATH_TO_ARTIFACTS: Path = PATH_TO_OUTPUTS.joinpath("artifacts") +PATH_TO_RESULTS: Path = PATH_TO_OUTPUTS.joinpath("results.json") +PATH_TO_LOGS: Path = PATH_TO_OUTPUTS.joinpath("logs.jsonl") + +# Streaming output paths for real-time progress +PATH_TO_PROGRESS: Path = PATH_TO_OUTPUTS.joinpath("progress.json") +PATH_TO_STREAM: Path = PATH_TO_OUTPUTS.joinpath("stream.jsonl") diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/exceptions.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/exceptions.py new file mode 100644 index 0000000..da1d040 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/exceptions.py @@ -0,0 +1,2 @@ +class FuzzForgeModuleError(Exception): + """TODO.""" diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/logs.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/logs.py new file mode 100644 index 0000000..d3a0fb4 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/logs.py @@ -0,0 +1,43 @@ +import logging +import sys + +import structlog + +from fuzzforge_modules_sdk.api.constants import PATH_TO_LOGS + + +class Formatter(logging.Formatter): + """TODO.""" + + def format(self, record: logging.LogRecord) -> str: + """TODO.""" + record.exc_info = None + return super().format(record) + + +def configure() -> None: + """TODO.""" + fmt: str = "%(message)s" + level = logging.DEBUG + PATH_TO_LOGS.parent.mkdir(exist_ok=True, parents=True) + PATH_TO_LOGS.unlink(missing_ok=True) + handler_file = logging.FileHandler(filename=PATH_TO_LOGS, mode="a") + handler_file.setFormatter(fmt=Formatter(fmt=fmt)) + handler_file.setLevel(level=level) + handler_stderr = logging.StreamHandler(stream=sys.stderr) + handler_stderr.setFormatter(fmt=Formatter(fmt=fmt)) + handler_stderr.setLevel(level=level) + logger: logging.Logger = logging.getLogger() + logger.setLevel(level=level) + logger.addHandler(handler_file) + logger.addHandler(handler_stderr) + structlog.configure( + processors=[ + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.dict_tracebacks, + structlog.processors.JSONRenderer(), + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + ) diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/models.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/models.py new file mode 100644 index 0000000..5bcabb8 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/models.py @@ -0,0 +1,85 @@ +from enum import StrEnum +from pathlib import Path # noqa: TC003 (required by pydantic at runtime) +from typing import TypeVar + +from pydantic import BaseModel, ConfigDict + + +class Base(BaseModel): + """TODO.""" + + model_config = ConfigDict(extra="forbid") + + +class FuzzForgeModulesSettingsBase(Base): + """TODO.""" + + +FuzzForgeModulesSettingsType = TypeVar("FuzzForgeModulesSettingsType", bound=FuzzForgeModulesSettingsBase) + + +class FuzzForgeModuleResources(StrEnum): + """Enumeration of artifact types.""" + + #: The type of the resource is unknown or irrelevant. + UNKNOWN = "unknown" + + +class FuzzForgeModuleResource(Base): + """TODO.""" + + #: The description of the resource. + description: str + #: The type of the resource. + kind: FuzzForgeModuleResources + #: The name of the resource. + name: str + #: The path of the resource on disk. + path: Path + + +class FuzzForgeModuleInputBase[FuzzForgeModulesSettingsType: FuzzForgeModulesSettingsBase](Base): + """The (standardized) input of a FuzzForge module.""" + + #: The collection of resources given to the module as inputs. + resources: list[FuzzForgeModuleResource] + #: The settings of the module. + settings: FuzzForgeModulesSettingsType + + +class FuzzForgeModuleArtifacts(StrEnum): + """Enumeration of artifact types.""" + + #: The artifact is an asset. + ASSET = "asset" + + +class FuzzForgeModuleArtifact(Base): + """An artifact generated by the module during its run.""" + + #: The description of the artifact. + description: str + #: The type of the artifact. + kind: FuzzForgeModuleArtifacts + #: The name of the artifact. + name: str + #: The path to the artifact on disk. + path: Path + + +class FuzzForgeModuleResults(StrEnum): + """TODO.""" + + SUCCESS = "success" + FAILURE = "failure" + + +class FuzzForgeModuleOutputBase(Base): + """The (standardized) output of a FuzzForge module.""" + + #: The collection of artifacts generated by the module during its run. + artifacts: list[FuzzForgeModuleArtifacts] + #: The path to the logs. + logs: Path + #: The result of the module's run. + result: FuzzForgeModuleResults diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/modules/__init__.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/modules/base.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/modules/base.py new file mode 100644 index 0000000..3cf89f9 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/api/modules/base.py @@ -0,0 +1,288 @@ +from abc import ABC, abstractmethod +import json +import time +from datetime import datetime, timezone +from shutil import rmtree +from typing import TYPE_CHECKING, Any, Final, final + +from structlog import get_logger + +from fuzzforge_modules_sdk.api.constants import ( + PATH_TO_ARTIFACTS, + PATH_TO_INPUT, + PATH_TO_LOGS, + PATH_TO_PROGRESS, + PATH_TO_RESULTS, + PATH_TO_STREAM, +) +from fuzzforge_modules_sdk.api.exceptions import FuzzForgeModuleError +from fuzzforge_modules_sdk.api.models import ( + FuzzForgeModuleArtifact, + FuzzForgeModuleArtifacts, + FuzzForgeModuleInputBase, + FuzzForgeModuleOutputBase, + FuzzForgeModuleResource, + FuzzForgeModuleResults, + FuzzForgeModulesSettingsType, +) + +if TYPE_CHECKING: + from pathlib import Path + + from structlog.stdlib import BoundLogger + + +class FuzzForgeModule(ABC): + """FuzzForge Modules' base.""" + + __artifacts: dict[str, FuzzForgeModuleArtifact] + + #: The logger associated with the module. + __logger: Final[BoundLogger] + + #: The name of the module. + __name: Final[str] + + #: The version of the module. + __version: Final[str] + + #: Start time for progress tracking. + __start_time: float + + #: Custom output data set by the module. + __output_data: dict[str, Any] + + def __init__(self, name: str, version: str) -> None: + """Initialize an instance of the class. + + :param name: The name of the module. + :param version: The version of the module. + + """ + self.__artifacts = {} + self.__logger = get_logger("module") + self.__name = name + self.__version = version + self.__start_time = time.time() + self.__output_data = {} + + # Initialize streaming output files + PATH_TO_PROGRESS.parent.mkdir(exist_ok=True, parents=True) + PATH_TO_STREAM.parent.mkdir(exist_ok=True, parents=True) + + @final + def get_logger(self) -> BoundLogger: + """Return the logger associated with the module.""" + return self.__logger + + @final + def get_name(self) -> str: + """Return the name of the module.""" + return self.__name + + @final + def get_version(self) -> str: + """Return the version of the module.""" + return self.__version + + @final + def set_output(self, **kwargs: Any) -> None: + """Set custom output data to be included in results.json. + + Call this from _run() to add module-specific fields to the output. + + :param kwargs: Key-value pairs to include in the output. + + Example: + self.set_output( + total_targets=4, + valid_targets=["target1", "target2"], + results=[...] + ) + + """ + self.__output_data.update(kwargs) + + @final + def emit_progress( + self, + progress: int, + status: str = "running", + message: str = "", + metrics: dict[str, Any] | None = None, + current_task: str = "", + ) -> None: + """Emit a progress update to the progress file. + + This method writes to /data/output/progress.json which can be polled + by the orchestrator or UI to show real-time progress. + + :param progress: Progress percentage (0-100). + :param status: Current status ("initializing", "running", "completed", "failed"). + :param message: Human-readable status message. + :param metrics: Dictionary of metrics (e.g., {"executions": 1000, "coverage": 50}). + :param current_task: Name of the current task being performed. + + """ + elapsed = time.time() - self.__start_time + + progress_data = { + "module": self.__name, + "version": self.__version, + "status": status, + "progress": max(0, min(100, progress)), + "message": message, + "current_task": current_task, + "elapsed_seconds": round(elapsed, 2), + "timestamp": datetime.now(timezone.utc).isoformat(), + "metrics": metrics or {}, + } + + PATH_TO_PROGRESS.write_text(json.dumps(progress_data, indent=2)) + + @final + def emit_event(self, event: str, **data: Any) -> None: + """Emit a streaming event to the stream file. + + This method appends to /data/output/stream.jsonl which can be tailed + by the orchestrator or UI for real-time event streaming. + + :param event: Event type (e.g., "crash_found", "target_started", "metrics"). + :param data: Additional event data as keyword arguments. + + """ + elapsed = time.time() - self.__start_time + + event_data = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "elapsed_seconds": round(elapsed, 2), + "module": self.__name, + "event": event, + **data, + } + + # Append to stream file (create if doesn't exist) + with PATH_TO_STREAM.open("a") as f: + f.write(json.dumps(event_data) + "\n") + + @final + def get_elapsed_seconds(self) -> float: + """Return the elapsed time since module start. + + :returns: Elapsed time in seconds. + + """ + return time.time() - self.__start_time + + @final + def _register_artifact(self, name: str, kind: FuzzForgeModuleArtifacts, description: str, path: Path) -> None: + """Register an artifact. + + :param name: The name of the artifact. + :param kind: The type of the artifact. + :param description: The description of the artifact. + :param path: The path of the artifact on the file system. + + """ + source: Path = path.resolve(strict=True) + destination: Path = PATH_TO_ARTIFACTS.joinpath(name).resolve() + if destination.parent != PATH_TO_ARTIFACTS: + message: str = f"path '{destination} is not a direct descendant of path '{PATH_TO_ARTIFACTS}'" + raise FuzzForgeModuleError(message) + if destination.exists(follow_symlinks=False): + if destination.is_file() or destination.is_symlink(): + destination.unlink() + elif destination.is_dir(): + rmtree(destination) + else: + message = f"unable to remove resource at path '{destination}': unsupported resource type" + raise FuzzForgeModuleError(message) + destination.parent.mkdir(exist_ok=True, parents=True) + source.copy(destination) + self.__artifacts[name] = FuzzForgeModuleArtifact( + description=description, + kind=kind, + name=name, + path=path, + ) + + @final + def main(self) -> None: + """TODO.""" + result = FuzzForgeModuleResults.SUCCESS + + try: + buffer: bytes = PATH_TO_INPUT.read_bytes() + data = self._get_input_type().model_validate_json(buffer) + self._prepare(settings=data.settings) + except: # noqa: E722 + self.get_logger().exception(event="exception during 'prepare' step") + result = FuzzForgeModuleResults.FAILURE + + if result != FuzzForgeModuleResults.FAILURE: + try: + result = self._run(resources=data.resources) + except: # noqa: E722 + self.get_logger().exception(event="exception during 'run' step") + result = FuzzForgeModuleResults.FAILURE + + if result != FuzzForgeModuleResults.FAILURE: + try: + self._cleanup(settings=data.settings) + except: # noqa: E722 + self.get_logger().exception(event="exception during 'cleanup' step") + + output = self._get_output_type()( + artifacts=list(self.__artifacts.values()), + logs=PATH_TO_LOGS, + result=result, + **self.__output_data, + ) + buffer = output.model_dump_json().encode("utf-8") + PATH_TO_RESULTS.parent.mkdir(exist_ok=True, parents=True) + PATH_TO_RESULTS.write_bytes(buffer) + + @classmethod + @abstractmethod + def _get_input_type(cls) -> type[FuzzForgeModuleInputBase[Any]]: + """TODO.""" + message: str = f"method '_get_input_type' is not implemented for class '{cls.__name__}'" + raise NotImplementedError(message) + + @classmethod + @abstractmethod + def _get_output_type(cls) -> type[FuzzForgeModuleOutputBase]: + """TODO.""" + message: str = f"method '_get_output_type' is not implemented for class '{cls.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def _prepare(self, settings: FuzzForgeModulesSettingsType) -> None: + """TODO. + + :param settings: TODO. + + """ + message: str = f"method '_prepare' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: + """TODO. + + :param resources: TODO. + :returns: TODO. + + """ + message: str = f"method '_run' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) + + @abstractmethod + def _cleanup(self, settings: FuzzForgeModulesSettingsType) -> None: + """TODO. + + :param settings: TODO. + + """ + message: str = f"method '_cleanup' is not implemented for class '{self.__class__.__name__}'" + raise NotImplementedError(message) diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/assets/Dockerfile b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/assets/Dockerfile new file mode 100644 index 0000000..416c8f2 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/assets/Dockerfile @@ -0,0 +1,20 @@ +FROM docker.io/debian:trixie as base + +COPY --from=ghcr.io/astral-sh/uv:0.9.10 /uv /uvx /bin/ + +FROM base as builder + +WORKDIR /sdk + +COPY ./src /sdk/src +COPY ./pyproject.toml /sdk/pyproject.toml + +RUN uv build --wheel -o /sdk/distributions + +FROM base as final + +COPY --from=builder /sdk/distributions /wheels + +WORKDIR /app + +CMD [ "/usr/bin/sleep", "infinity" ] diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/assets/pyproject.toml b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/assets/pyproject.toml new file mode 120000 index 0000000..7aa7944 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/assets/pyproject.toml @@ -0,0 +1 @@ +../../../pyproject.toml \ No newline at end of file diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/py.typed b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/Dockerfile b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/Dockerfile new file mode 100644 index 0000000..222f061 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/Dockerfile @@ -0,0 +1,9 @@ +FROM localhost/fuzzforge-modules-sdk:0.0.1 + +COPY ./src /app/src +COPY ./pyproject.toml /app/pyproject.toml + +# Remove workspace reference since we're using wheels +RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml + +RUN uv sync --find-links /wheels diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/Makefile b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/Makefile new file mode 100644 index 0000000..cada4d0 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/Makefile @@ -0,0 +1,45 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit build clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +build: + $(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION) + +save: build + $(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/README.md b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/README.md new file mode 100644 index 0000000..d0671a1 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/README.md @@ -0,0 +1,46 @@ +# FuzzForge Modules - FIXME + +## Installation + +### Python + +```shell +# install the package (users) +uv sync +# install the package and all development dependencies (developers) +uv sync --all-extras +``` + +### Container + +```shell +# build the image +make build +# run the container +mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output" +echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json" +podman run --rm \ + --volume "${PWD}/data:/data" \ + ':' 'uv run module' +``` + +## Usage + +```shell +uv run module +``` + +## Development tools + +```shell +# run ruff (formatter) +make format +# run mypy (type checker) +make mypy +# run tests (pytest) +make pytest +# run ruff (linter) +make ruff +``` + +See the file `Makefile` at the root of this directory for more tools. diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/mypy.ini b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/pyproject.toml b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/pyproject.toml new file mode 100644 index 0000000..29dea4b --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "fuzzforge-module-template" +version = "0.0.1" +description = "FIXME" +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-modules-sdk==0.0.1", + "pydantic==2.12.4", + "structlog==25.5.0", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +module = "module.__main__:main" + +[tool.uv.sources] +fuzzforge-modules-sdk = { workspace = true } + +[tool.uv] +package = true diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/ruff.toml b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/__init__.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/__main__.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/__main__.py new file mode 100644 index 0000000..bc8914a --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/__main__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api import logs + +from module.mod import Module + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + + +def main() -> None: + """TODO.""" + logs.configure() + module: FuzzForgeModule = Module() + module.main() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/mod.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/mod.py new file mode 100644 index 0000000..f0f85e9 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/mod.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults +from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + +from module.models import Input, Output + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource, FuzzForgeModulesSettingsType + + +class Module(FuzzForgeModule): + """TODO.""" + + def __init__(self) -> None: + """Initialize an instance of the class.""" + name: str = "FIXME" + version: str = "FIXME" + FuzzForgeModule.__init__(self, name=name, version=version) + + @classmethod + def _get_input_type(cls) -> type[Input]: + """TODO.""" + return Input + + @classmethod + def _get_output_type(cls) -> type[Output]: + """TODO.""" + return Output + + def _prepare(self, settings: FuzzForgeModulesSettingsType) -> None: + """TODO. + + :param settings: TODO. + + """ + + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: # noqa: ARG002 + """TODO. + + :param resources: TODO. + :returns: TODO. + + """ + return FuzzForgeModuleResults.SUCCESS + + def _cleanup(self, settings: FuzzForgeModulesSettingsType) -> None: + """TODO. + + :param settings: TODO. + + """ diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/models.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/models.py new file mode 100644 index 0000000..2a3f021 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/models.py @@ -0,0 +1,11 @@ +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase + +from module.settings import Settings + + +class Input(FuzzForgeModuleInputBase[Settings]): + """TODO.""" + + +class Output(FuzzForgeModuleOutputBase): + """TODO.""" diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/settings.py b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/settings.py new file mode 100644 index 0000000..f916ad4 --- /dev/null +++ b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/src/module/settings.py @@ -0,0 +1,7 @@ +from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase + + +class Settings(FuzzForgeModulesSettingsBase): + """TODO.""" + + # Here goes your attributes diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/tests/.gitkeep b/fuzzforge-modules/fuzzforge-modules-sdk/src/fuzzforge_modules_sdk/templates/fuzzforge-module-template/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/fuzzforge-modules-sdk/tests/.gitkeep b/fuzzforge-modules/fuzzforge-modules-sdk/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/harness-validator/Dockerfile b/fuzzforge-modules/harness-validator/Dockerfile new file mode 100644 index 0000000..c75995b --- /dev/null +++ b/fuzzforge-modules/harness-validator/Dockerfile @@ -0,0 +1,23 @@ +FROM localhost/fuzzforge-modules-sdk:0.1.0 + +# Install build tools and Rust nightly for compiling fuzz harnesses +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install cargo-fuzz for validation +RUN cargo install cargo-fuzz --locked || true + +COPY ./src /app/src +COPY ./pyproject.toml /app/pyproject.toml + +# Remove workspace reference since we're using wheels +RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml + +RUN uv sync --find-links /wheels diff --git a/fuzzforge-modules/harness-validator/Makefile b/fuzzforge-modules/harness-validator/Makefile new file mode 100644 index 0000000..cada4d0 --- /dev/null +++ b/fuzzforge-modules/harness-validator/Makefile @@ -0,0 +1,45 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit build clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +build: + $(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION) + +save: build + $(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/harness-validator/README.md b/fuzzforge-modules/harness-validator/README.md new file mode 100644 index 0000000..d0671a1 --- /dev/null +++ b/fuzzforge-modules/harness-validator/README.md @@ -0,0 +1,46 @@ +# FuzzForge Modules - FIXME + +## Installation + +### Python + +```shell +# install the package (users) +uv sync +# install the package and all development dependencies (developers) +uv sync --all-extras +``` + +### Container + +```shell +# build the image +make build +# run the container +mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output" +echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json" +podman run --rm \ + --volume "${PWD}/data:/data" \ + ':' 'uv run module' +``` + +## Usage + +```shell +uv run module +``` + +## Development tools + +```shell +# run ruff (formatter) +make format +# run mypy (type checker) +make mypy +# run tests (pytest) +make pytest +# run ruff (linter) +make ruff +``` + +See the file `Makefile` at the root of this directory for more tools. diff --git a/fuzzforge-modules/harness-validator/mypy.ini b/fuzzforge-modules/harness-validator/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-modules/harness-validator/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/harness-validator/pyproject.toml b/fuzzforge-modules/harness-validator/pyproject.toml new file mode 100644 index 0000000..d3c8d1f --- /dev/null +++ b/fuzzforge-modules/harness-validator/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "harness-validator" +version = "0.1.0" +description = "FuzzForge module that validates fuzz harnesses compile correctly" +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-modules-sdk==0.0.1", + "pydantic==2.12.4", + "structlog==25.5.0", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +module = "module.__main__:main" + +[tool.uv.sources] +fuzzforge-modules-sdk = { workspace = true } + +[tool.uv] +package = true diff --git a/fuzzforge-modules/harness-validator/ruff.toml b/fuzzforge-modules/harness-validator/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/harness-validator/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/harness-validator/src/module/__init__.py b/fuzzforge-modules/harness-validator/src/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/harness-validator/src/module/__main__.py b/fuzzforge-modules/harness-validator/src/module/__main__.py new file mode 100644 index 0000000..bc8914a --- /dev/null +++ b/fuzzforge-modules/harness-validator/src/module/__main__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api import logs + +from module.mod import Module + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + + +def main() -> None: + """TODO.""" + logs.configure() + module: FuzzForgeModule = Module() + module.main() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-modules/harness-validator/src/module/mod.py b/fuzzforge-modules/harness-validator/src/module/mod.py new file mode 100644 index 0000000..8766ac6 --- /dev/null +++ b/fuzzforge-modules/harness-validator/src/module/mod.py @@ -0,0 +1,309 @@ +"""Harness Validator module for FuzzForge. + +This module validates that fuzz harnesses compile correctly. +It takes a Rust project with a fuzz directory containing harnesses +and runs cargo build to verify they compile. +""" + +from __future__ import annotations + +import json +import subprocess +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import structlog + +from fuzzforge_modules_sdk.api.constants import PATH_TO_INPUTS, PATH_TO_OUTPUTS +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults +from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + +from module.models import Input, Output, ValidationResult, HarnessStatus +from module.settings import Settings + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource + +logger = structlog.get_logger() + + +class Module(FuzzForgeModule): + """Harness Validator module - validates that fuzz harnesses compile.""" + + _settings: Settings | None + _results: list[ValidationResult] + + def __init__(self) -> None: + """Initialize an instance of the class.""" + name: str = "harness-validator" + version: str = "0.1.0" + FuzzForgeModule.__init__(self, name=name, version=version) + self._settings = None + self._results = [] + + @classmethod + def _get_input_type(cls) -> type[Input]: + """Return the input type.""" + return Input + + @classmethod + def _get_output_type(cls) -> type[Output]: + """Return the output type.""" + return Output + + def _prepare(self, settings: Settings) -> None: # type: ignore[override] + """Prepare the module. + + :param settings: Module settings. + + """ + self._settings = settings + logger.info("harness-validator preparing", settings=settings.model_dump() if settings else {}) + + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: + """Run the harness validator. + + :param resources: Input resources (fuzz project directory). + :returns: Module execution result. + + """ + logger.info("harness-validator starting", resource_count=len(resources)) + + # Find the fuzz project directory + fuzz_project_src = self._find_fuzz_project(resources) + if fuzz_project_src is None: + logger.error("No fuzz project found in resources") + return FuzzForgeModuleResults.FAILURE + + logger.info("Found fuzz project", path=str(fuzz_project_src)) + + # Copy the project to a writable location since /data/input is read-only + # and cargo needs to write Cargo.lock and build artifacts + import shutil + work_dir = Path("/tmp/fuzz-build") + if work_dir.exists(): + shutil.rmtree(work_dir) + + # Copy entire project root (parent of fuzz directory) + project_root = fuzz_project_src.parent + work_project = work_dir / project_root.name + shutil.copytree(project_root, work_project, dirs_exist_ok=True) + + # Adjust fuzz_project to point to the copied location + fuzz_project = work_dir / project_root.name / fuzz_project_src.name + logger.info("Copied project to writable location", work_dir=str(fuzz_project)) + + # Find all harness targets + targets = self._find_harness_targets(fuzz_project) + if not targets: + logger.error("No harness targets found") + return FuzzForgeModuleResults.FAILURE + + logger.info("Found harness targets", count=len(targets)) + + # Validate each harness + all_valid = True + for target in targets: + result = self._validate_harness(fuzz_project, target) + self._results.append(result) + if result.status != HarnessStatus.VALID: + all_valid = False + logger.warning("Harness validation failed", + target=target, + status=result.status.value, + errors=result.errors) + else: + logger.info("Harness valid", target=target) + + # Set output data for results.json + valid_targets = [r.target for r in self._results if r.status == HarnessStatus.VALID] + invalid_targets = [r.target for r in self._results if r.status != HarnessStatus.VALID] + + self.set_output( + fuzz_project=str(fuzz_project), + total_targets=len(self._results), + valid_count=len(valid_targets), + invalid_count=len(invalid_targets), + valid_targets=valid_targets, + invalid_targets=invalid_targets, + results=[r.model_dump() for r in self._results], + ) + + valid_count = sum(1 for r in self._results if r.status == HarnessStatus.VALID) + logger.info("harness-validator completed", + total=len(self._results), + valid=valid_count, + invalid=len(self._results) - valid_count) + + return FuzzForgeModuleResults.SUCCESS + + def _cleanup(self, settings: Settings) -> None: # type: ignore[override] + """Clean up after execution. + + :param settings: Module settings. + + """ + pass + + def _find_fuzz_project(self, resources: list[FuzzForgeModuleResource]) -> Path | None: + """Find the fuzz project directory in the resources. + + :param resources: List of input resources. + :returns: Path to fuzz project or None. + + """ + for resource in resources: + path = Path(resource.path) + + # Check if it's a fuzz directory with Cargo.toml + if path.is_dir(): + cargo_toml = path / "Cargo.toml" + if cargo_toml.exists(): + # Check if it has fuzz_targets directory + fuzz_targets = path / "fuzz_targets" + if fuzz_targets.is_dir(): + return path + + # Check for fuzz subdirectory + fuzz_dir = path / "fuzz" + if fuzz_dir.is_dir(): + cargo_toml = fuzz_dir / "Cargo.toml" + if cargo_toml.exists(): + return fuzz_dir + + return None + + def _find_harness_targets(self, fuzz_project: Path) -> list[str]: + """Find all harness target names in the fuzz project. + + :param fuzz_project: Path to the fuzz project. + :returns: List of target names. + + """ + targets = [] + fuzz_targets_dir = fuzz_project / "fuzz_targets" + + if fuzz_targets_dir.is_dir(): + for rs_file in fuzz_targets_dir.glob("*.rs"): + # Target name is the file name without extension + target_name = rs_file.stem + targets.append(target_name) + + return targets + + def _validate_harness(self, fuzz_project: Path, target: str) -> ValidationResult: + """Validate a single harness by compiling it. + + :param fuzz_project: Path to the fuzz project. + :param target: Name of the harness target. + :returns: Validation result. + + """ + harness_file = fuzz_project / "fuzz_targets" / f"{target}.rs" + + if not harness_file.exists(): + return ValidationResult( + target=target, + file_path=str(harness_file), + status=HarnessStatus.NOT_FOUND, + errors=["Harness file not found"], + ) + + # Try to compile just this target + try: + env = os.environ.copy() + env["CARGO_INCREMENTAL"] = "0" + + result = subprocess.run( + [ + "cargo", "build", + "--bin", target, + "--message-format=json", + ], + cwd=fuzz_project, + capture_output=True, + text=True, + timeout=self._settings.compile_timeout if self._settings else 120, + env=env, + ) + + # Parse cargo output for errors + errors = [] + warnings = [] + + for line in result.stdout.splitlines(): + try: + msg = json.loads(line) + if msg.get("reason") == "compiler-message": + message = msg.get("message", {}) + level = message.get("level", "") + rendered = message.get("rendered", "") + + if level == "error": + errors.append(rendered.strip()) + elif level == "warning": + warnings.append(rendered.strip()) + except json.JSONDecodeError: + pass + + # Also check stderr for any cargo errors + if result.returncode != 0 and not errors: + errors.append(result.stderr.strip() if result.stderr else "Build failed with unknown error") + + if result.returncode == 0: + return ValidationResult( + target=target, + file_path=str(harness_file), + status=HarnessStatus.VALID, + errors=[], + warnings=warnings, + ) + else: + return ValidationResult( + target=target, + file_path=str(harness_file), + status=HarnessStatus.COMPILE_ERROR, + errors=errors, + warnings=warnings, + ) + + except subprocess.TimeoutExpired: + return ValidationResult( + target=target, + file_path=str(harness_file), + status=HarnessStatus.TIMEOUT, + errors=["Compilation timed out"], + ) + except Exception as e: + return ValidationResult( + target=target, + file_path=str(harness_file), + status=HarnessStatus.ERROR, + errors=[str(e)], + ) + + def _write_output(self, fuzz_project: Path) -> None: + """Write the validation results to output. + + :param fuzz_project: Path to the fuzz project. + + """ + output_path = PATH_TO_OUTPUTS / "validation.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + + valid_targets = [r.target for r in self._results if r.status == HarnessStatus.VALID] + invalid_targets = [r.target for r in self._results if r.status != HarnessStatus.VALID] + + output_data = { + "fuzz_project": str(fuzz_project), + "total_targets": len(self._results), + "valid_count": len(valid_targets), + "invalid_count": len(invalid_targets), + "valid_targets": valid_targets, + "invalid_targets": invalid_targets, + "results": [r.model_dump() for r in self._results], + } + + output_path.write_text(json.dumps(output_data, indent=2)) + logger.info("wrote validation results", path=str(output_path)) diff --git a/fuzzforge-modules/harness-validator/src/module/models.py b/fuzzforge-modules/harness-validator/src/module/models.py new file mode 100644 index 0000000..1a52e55 --- /dev/null +++ b/fuzzforge-modules/harness-validator/src/module/models.py @@ -0,0 +1,71 @@ +"""Models for the harness-validator module.""" + +from enum import Enum + +from pydantic import BaseModel, Field +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase + +from module.settings import Settings + + +class HarnessStatus(str, Enum): + """Status of harness validation.""" + + VALID = "valid" + COMPILE_ERROR = "compile_error" + NOT_FOUND = "not_found" + TIMEOUT = "timeout" + ERROR = "error" + + +class ValidationResult(BaseModel): + """Result of validating a single harness.""" + + #: Name of the harness target + target: str + + #: Path to the harness file + file_path: str + + #: Validation status + status: HarnessStatus + + #: Compilation errors (if any) + errors: list[str] = Field(default_factory=list) + + #: Compilation warnings (if any) + warnings: list[str] = Field(default_factory=list) + + +class Input(FuzzForgeModuleInputBase[Settings]): + """Input for the harness-validator module. + + Expects a fuzz project directory with: + - Cargo.toml + - fuzz_targets/ directory with .rs harness files + """ + + +class Output(FuzzForgeModuleOutputBase): + """Output from the harness-validator module.""" + + #: Path to the fuzz project + fuzz_project: str = "" + + #: Total number of harness targets + total_targets: int = 0 + + #: Number of valid (compilable) harnesses + valid_count: int = 0 + + #: Number of invalid harnesses + invalid_count: int = 0 + + #: List of valid target names (ready for fuzzing) + valid_targets: list[str] = Field(default_factory=list) + + #: List of invalid target names (need fixes) + invalid_targets: list[str] = Field(default_factory=list) + + #: Detailed validation results per target + results: list[ValidationResult] = Field(default_factory=list) diff --git a/fuzzforge-modules/harness-validator/src/module/settings.py b/fuzzforge-modules/harness-validator/src/module/settings.py new file mode 100644 index 0000000..aeab815 --- /dev/null +++ b/fuzzforge-modules/harness-validator/src/module/settings.py @@ -0,0 +1,13 @@ +"""Settings for the harness-validator module.""" + +from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase + + +class Settings(FuzzForgeModulesSettingsBase): + """Settings for the harness-validator module.""" + + #: Timeout for compiling each harness (seconds) + compile_timeout: int = 120 + + #: Whether to stop on first error + fail_fast: bool = False diff --git a/fuzzforge-modules/harness-validator/tests/.gitkeep b/fuzzforge-modules/harness-validator/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/rust-analyzer/Dockerfile b/fuzzforge-modules/rust-analyzer/Dockerfile new file mode 100644 index 0000000..c88ac3a --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/Dockerfile @@ -0,0 +1,25 @@ +FROM localhost/fuzzforge-modules-sdk:0.1.0 + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust toolchain +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install Rust analysis tools (skipping cargo-geiger as it's heavy) +# RUN cargo install cargo-geiger --locked || true +RUN cargo install cargo-audit --locked || true + +COPY ./src /app/src +COPY ./pyproject.toml /app/pyproject.toml + +# Remove workspace reference since we're using wheels +RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml + +RUN uv sync --find-links /wheels diff --git a/fuzzforge-modules/rust-analyzer/Makefile b/fuzzforge-modules/rust-analyzer/Makefile new file mode 100644 index 0000000..cada4d0 --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/Makefile @@ -0,0 +1,45 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +PODMAN?=/usr/bin/podman + +SOURCES=./src +TESTS=./tests + +.PHONY: bandit build clean format mypy pytest ruff version + +bandit: + uv run bandit --recursive $(SOURCES) + +build: + $(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION) + +save: build + $(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION) + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) $(TESTS) + +mypy: + uv run mypy $(SOURCES) + +pytest: + uv run pytest $(TESTS) + +ruff: + uv run ruff check --fix $(SOURCES) $(TESTS) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-modules/rust-analyzer/README.md b/fuzzforge-modules/rust-analyzer/README.md new file mode 100644 index 0000000..d0671a1 --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/README.md @@ -0,0 +1,46 @@ +# FuzzForge Modules - FIXME + +## Installation + +### Python + +```shell +# install the package (users) +uv sync +# install the package and all development dependencies (developers) +uv sync --all-extras +``` + +### Container + +```shell +# build the image +make build +# run the container +mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output" +echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json" +podman run --rm \ + --volume "${PWD}/data:/data" \ + ':' 'uv run module' +``` + +## Usage + +```shell +uv run module +``` + +## Development tools + +```shell +# run ruff (formatter) +make format +# run mypy (type checker) +make mypy +# run tests (pytest) +make pytest +# run ruff (linter) +make ruff +``` + +See the file `Makefile` at the root of this directory for more tools. diff --git a/fuzzforge-modules/rust-analyzer/mypy.ini b/fuzzforge-modules/rust-analyzer/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-modules/rust-analyzer/pyproject.toml b/fuzzforge-modules/rust-analyzer/pyproject.toml new file mode 100644 index 0000000..eb07920 --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "rust-analyzer" +version = "0.0.1" +description = "FIXME" +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-modules-sdk==0.0.1", + "pydantic==2.12.4", + "structlog==25.5.0", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] +tests = [ + "pytest==9.0.2", +] + +[project.scripts] +module = "module.__main__:main" + +[tool.uv] +package = true diff --git a/fuzzforge-modules/rust-analyzer/ruff.toml b/fuzzforge-modules/rust-analyzer/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-modules/rust-analyzer/src/module/__init__.py b/fuzzforge-modules/rust-analyzer/src/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-modules/rust-analyzer/src/module/__main__.py b/fuzzforge-modules/rust-analyzer/src/module/__main__.py new file mode 100644 index 0000000..bc8914a --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/src/module/__main__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api import logs + +from module.mod import Module + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + + +def main() -> None: + """TODO.""" + logs.configure() + module: FuzzForgeModule = Module() + module.main() + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-modules/rust-analyzer/src/module/mod.py b/fuzzforge-modules/rust-analyzer/src/module/mod.py new file mode 100644 index 0000000..751d3bd --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/src/module/mod.py @@ -0,0 +1,314 @@ +"""Rust Analyzer module for FuzzForge. + +This module analyzes Rust source code to identify fuzzable entry points, +unsafe blocks, and known vulnerabilities. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +from fuzzforge_modules_sdk.api.constants import PATH_TO_OUTPUTS +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults +from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule + +from module.models import AnalysisResult, EntryPoint, Input, Output, UnsafeBlock, Vulnerability +from module.settings import Settings + +if TYPE_CHECKING: + from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource + + +class Module(FuzzForgeModule): + """Rust Analyzer module - analyzes Rust code for fuzzable entry points.""" + + def __init__(self) -> None: + """Initialize an instance of the class.""" + name: str = "rust-analyzer" + version: str = "0.1.0" + FuzzForgeModule.__init__(self, name=name, version=version) + self._project_path: Path | None = None + self._settings: Settings | None = None + + @classmethod + def _get_input_type(cls) -> type[Input]: + """Return the input type.""" + return Input + + @classmethod + def _get_output_type(cls) -> type[Output]: + """Return the output type.""" + return Output + + def _prepare(self, settings: Settings) -> None: # type: ignore[override] + """Prepare the module. + + :param settings: Module settings. + + """ + self._settings = settings + + def _find_cargo_toml(self, resources: list[FuzzForgeModuleResource]) -> Path | None: + """Find the Cargo.toml file in the resources. + + :param resources: List of input resources. + :returns: Path to Cargo.toml or None. + + """ + for resource in resources: + if resource.path.name == "Cargo.toml": + return resource.path + # Check if resource is a directory containing Cargo.toml + cargo_path = resource.path / "Cargo.toml" + if cargo_path.exists(): + return cargo_path + return None + + def _parse_cargo_toml(self, cargo_path: Path) -> tuple[str, str, str]: + """Parse Cargo.toml to extract crate name, version, and lib name. + + :param cargo_path: Path to Cargo.toml. + :returns: Tuple of (crate_name, version, lib_name). + + """ + import tomllib + + with cargo_path.open("rb") as f: + data = tomllib.load(f) + + package = data.get("package", {}) + crate_name = package.get("name", "unknown") + version = package.get("version", "0.0.0") + + # Get lib name - defaults to crate name with dashes converted to underscores + lib_section = data.get("lib", {}) + lib_name = lib_section.get("name", crate_name.replace("-", "_")) + + return crate_name, version, lib_name + + def _find_entry_points(self, project_path: Path) -> list[EntryPoint]: + """Find fuzzable entry points in the Rust source. + + :param project_path: Path to the Rust project. + :returns: List of entry points. + + """ + entry_points: list[EntryPoint] = [] + + # Patterns for fuzzable functions (take &[u8], &str, or impl Read) + fuzzable_patterns = [ + r"pub\s+fn\s+(\w+)\s*\([^)]*&\[u8\][^)]*\)", + r"pub\s+fn\s+(\w+)\s*\([^)]*&str[^)]*\)", + r"pub\s+fn\s+(\w+)\s*\([^)]*impl\s+Read[^)]*\)", + r"pub\s+fn\s+(\w+)\s*\([^)]*data:\s*&\[u8\][^)]*\)", + r"pub\s+fn\s+(\w+)\s*\([^)]*input:\s*&\[u8\][^)]*\)", + r"pub\s+fn\s+(\w+)\s*\([^)]*buf:\s*&\[u8\][^)]*\)", + ] + + # Also find parse/decode functions + parser_patterns = [ + r"pub\s+fn\s+(parse\w*)\s*\([^)]*\)", + r"pub\s+fn\s+(decode\w*)\s*\([^)]*\)", + r"pub\s+fn\s+(deserialize\w*)\s*\([^)]*\)", + r"pub\s+fn\s+(from_bytes\w*)\s*\([^)]*\)", + r"pub\s+fn\s+(read\w*)\s*\([^)]*\)", + ] + + src_path = project_path / "src" + if not src_path.exists(): + src_path = project_path + + for rust_file in src_path.rglob("*.rs"): + try: + content = rust_file.read_text() + lines = content.split("\n") + + for line_num, line in enumerate(lines, 1): + # Check fuzzable patterns + for pattern in fuzzable_patterns: + match = re.search(pattern, line) + if match: + entry_points.append( + EntryPoint( + function=match.group(1), + file=str(rust_file.relative_to(project_path)), + line=line_num, + signature=line.strip(), + fuzzable=True, + ) + ) + + # Check parser patterns (may need manual review) + for pattern in parser_patterns: + match = re.search(pattern, line) + if match: + # Avoid duplicates + func_name = match.group(1) + if not any(ep.function == func_name for ep in entry_points): + entry_points.append( + EntryPoint( + function=func_name, + file=str(rust_file.relative_to(project_path)), + line=line_num, + signature=line.strip(), + fuzzable=True, + ) + ) + except Exception: + continue + + return entry_points + + def _find_unsafe_blocks(self, project_path: Path) -> list[UnsafeBlock]: + """Find unsafe blocks in the Rust source. + + :param project_path: Path to the Rust project. + :returns: List of unsafe blocks. + + """ + unsafe_blocks: list[UnsafeBlock] = [] + + src_path = project_path / "src" + if not src_path.exists(): + src_path = project_path + + for rust_file in src_path.rglob("*.rs"): + try: + content = rust_file.read_text() + lines = content.split("\n") + + for line_num, line in enumerate(lines, 1): + if "unsafe" in line and ("{" in line or "fn" in line): + # Determine context + context = "unsafe block" + if "unsafe fn" in line: + context = "unsafe function" + elif "unsafe impl" in line: + context = "unsafe impl" + elif "*const" in line or "*mut" in line: + context = "raw pointer operation" + + unsafe_blocks.append( + UnsafeBlock( + file=str(rust_file.relative_to(project_path)), + line=line_num, + context=context, + ) + ) + except Exception: + continue + + return unsafe_blocks + + def _run_cargo_audit(self, project_path: Path) -> list[Vulnerability]: + """Run cargo-audit to find known vulnerabilities. + + :param project_path: Path to the Rust project. + :returns: List of vulnerabilities. + + """ + vulnerabilities: list[Vulnerability] = [] + + try: + result = subprocess.run( + ["cargo", "audit", "--json"], + cwd=project_path, + capture_output=True, + text=True, + timeout=120, + ) + + if result.stdout: + audit_data = json.loads(result.stdout) + for vuln in audit_data.get("vulnerabilities", {}).get("list", []): + advisory = vuln.get("advisory", {}) + vulnerabilities.append( + Vulnerability( + advisory_id=advisory.get("id", "UNKNOWN"), + crate_name=vuln.get("package", {}).get("name", "unknown"), + version=vuln.get("package", {}).get("version", "0.0.0"), + title=advisory.get("title", "Unknown vulnerability"), + severity=advisory.get("severity", "unknown"), + ) + ) + except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError): + pass + + return vulnerabilities + + def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: + """Run the analysis. + + :param resources: Input resources. + :returns: Module result status. + + """ + # Find the Rust project + cargo_path = self._find_cargo_toml(resources) + if cargo_path is None: + self.get_logger().error("No Cargo.toml found in resources") + return FuzzForgeModuleResults.FAILURE + + project_path = cargo_path.parent + self._project_path = project_path + + self.get_logger().info("Analyzing Rust project", project=str(project_path)) + + # Parse Cargo.toml + crate_name, crate_version, lib_name = self._parse_cargo_toml(cargo_path) + self.get_logger().info("Found crate", name=crate_name, version=crate_version, lib_name=lib_name) + + # Find entry points + entry_points = self._find_entry_points(project_path) + self.get_logger().info("Found entry points", count=len(entry_points)) + + # Find unsafe blocks + unsafe_blocks = self._find_unsafe_blocks(project_path) + self.get_logger().info("Found unsafe blocks", count=len(unsafe_blocks)) + + # Run cargo-audit if enabled + vulnerabilities: list[Vulnerability] = [] + if self._settings and self._settings.run_audit: + vulnerabilities = self._run_cargo_audit(project_path) + self.get_logger().info("Found vulnerabilities", count=len(vulnerabilities)) + + # Build result + analysis = AnalysisResult( + crate_name=crate_name, + crate_version=crate_version, + lib_name=lib_name, + entry_points=entry_points, + unsafe_blocks=unsafe_blocks, + vulnerabilities=vulnerabilities, + summary={ + "entry_points": len(entry_points), + "unsafe_blocks": len(unsafe_blocks), + "vulnerabilities": len(vulnerabilities), + }, + ) + + # Set output data for results.json + self.set_output( + analysis=analysis.model_dump(), + ) + + # Write analysis to output file (for backwards compatibility) + output_path = PATH_TO_OUTPUTS / "analysis.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(analysis.model_dump_json(indent=2)) + + self.get_logger().info("Analysis complete", output=str(output_path)) + + return FuzzForgeModuleResults.SUCCESS + + def _cleanup(self, settings: Settings) -> None: # type: ignore[override] + """Clean up after execution. + + :param settings: Module settings. + + """ + pass diff --git a/fuzzforge-modules/rust-analyzer/src/module/models.py b/fuzzforge-modules/rust-analyzer/src/module/models.py new file mode 100644 index 0000000..f87f280 --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/src/module/models.py @@ -0,0 +1,100 @@ +"""Models for rust-analyzer module.""" + +from pathlib import Path + +from pydantic import BaseModel + +from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase + +from module.settings import Settings + + +class Input(FuzzForgeModuleInputBase[Settings]): + """Input for the rust-analyzer module.""" + + +class EntryPoint(BaseModel): + """A fuzzable entry point in the Rust codebase.""" + + #: Function name. + function: str + + #: Source file path. + file: str + + #: Line number. + line: int + + #: Function signature. + signature: str + + #: Whether the function takes &[u8] or similar fuzzable input. + fuzzable: bool = True + + +class UnsafeBlock(BaseModel): + """An unsafe block detected in the codebase.""" + + #: Source file path. + file: str + + #: Line number. + line: int + + #: Context description. + context: str + + +class Vulnerability(BaseModel): + """A known vulnerability from cargo-audit.""" + + #: Advisory ID (e.g., RUSTSEC-2021-0001). + advisory_id: str + + #: Affected crate name. + crate_name: str + + #: Affected version. + version: str + + #: Vulnerability title. + title: str + + #: Severity level. + severity: str + + +class AnalysisResult(BaseModel): + """The complete analysis result.""" + + #: Crate name from Cargo.toml (use this in fuzz/Cargo.toml dependencies). + crate_name: str + + #: Crate version. + crate_version: str + + #: Library name for use in Rust code (use in `use` statements). + #: In Rust, dashes become underscores: "fuzz-demo" -> "fuzz_demo". + lib_name: str = "" + + #: List of fuzzable entry points. + entry_points: list[EntryPoint] + + #: List of unsafe blocks. + unsafe_blocks: list[UnsafeBlock] + + #: List of known vulnerabilities. + vulnerabilities: list[Vulnerability] + + #: Summary statistics. + summary: dict[str, int] + + +class Output(FuzzForgeModuleOutputBase): + """Output for the rust-analyzer module.""" + + #: The analysis result (as dict for serialization). + analysis: dict | None = None + + #: Path to the analysis JSON file. + analysis_file: Path | None = None diff --git a/fuzzforge-modules/rust-analyzer/src/module/settings.py b/fuzzforge-modules/rust-analyzer/src/module/settings.py new file mode 100644 index 0000000..17767ff --- /dev/null +++ b/fuzzforge-modules/rust-analyzer/src/module/settings.py @@ -0,0 +1,16 @@ +"""Settings for rust-analyzer module.""" + +from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase + + +class Settings(FuzzForgeModulesSettingsBase): + """Settings for the rust-analyzer module.""" + + #: Whether to run cargo-audit for CVE detection. + run_audit: bool = True + + #: Whether to run cargo-geiger for unsafe detection. + run_geiger: bool = True + + #: Maximum depth for dependency analysis. + max_depth: int = 3 diff --git a/fuzzforge-modules/rust-analyzer/tests/.gitkeep b/fuzzforge-modules/rust-analyzer/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-runner/Makefile b/fuzzforge-runner/Makefile new file mode 100644 index 0000000..eb78637 --- /dev/null +++ b/fuzzforge-runner/Makefile @@ -0,0 +1,14 @@ +.PHONY: lint test format check + +lint: + uv run ruff check src tests + uv run mypy src + +test: + uv run pytest tests + +format: + uv run ruff format src tests + uv run ruff check --fix src tests + +check: lint test diff --git a/fuzzforge-runner/README.md b/fuzzforge-runner/README.md new file mode 100644 index 0000000..570abdd --- /dev/null +++ b/fuzzforge-runner/README.md @@ -0,0 +1,44 @@ +# FuzzForge Runner + +Direct execution engine for FuzzForge OSS. Provides simplified module and workflow execution without requiring Temporal or external infrastructure. + +## Overview + +The Runner is designed for local-first operation, executing FuzzForge modules directly in containerized sandboxes (Docker/Podman) without workflow orchestration overhead. + +## Features + +- Direct module execution in isolated containers +- Sequential workflow orchestration (no Temporal required) +- Local filesystem storage (S3 optional) +- SQLite-based state management (optional) + +## Usage + +```python +from fuzzforge_runner import Runner +from fuzzforge_runner.settings import Settings + +settings = Settings() +runner = Runner(settings) + +# Execute a single module +result = await runner.execute_module( + module_identifier="my-module", + project_path="/path/to/project", +) + +# Execute a workflow (sequential steps) +result = await runner.execute_workflow( + workflow_definition=workflow, + project_path="/path/to/project", +) +``` + +## Configuration + +Environment variables: + +- `FUZZFORGE_STORAGE_PATH`: Local storage directory (default: `~/.fuzzforge/storage`) +- `FUZZFORGE_ENGINE_TYPE`: Container engine (`docker` or `podman`) +- `FUZZFORGE_ENGINE_SOCKET`: Container socket path diff --git a/fuzzforge-runner/mypy.ini b/fuzzforge-runner/mypy.ini new file mode 100644 index 0000000..be0671c --- /dev/null +++ b/fuzzforge-runner/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +strict = true diff --git a/fuzzforge-runner/pyproject.toml b/fuzzforge-runner/pyproject.toml new file mode 100644 index 0000000..f7d7caa --- /dev/null +++ b/fuzzforge-runner/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "fuzzforge-runner" +version = "0.0.1" +description = "FuzzForge Runner - Direct execution engine for FuzzForge OSS." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fuzzforge-common", + "fuzzforge-types", + "structlog>=25.5.0", + "pydantic>=2.12.4", + "pydantic-settings>=2.8.1", +] + +[project.scripts] +fuzzforge-runner = "fuzzforge_runner.__main__:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/fuzzforge_runner"] + +[tool.uv.sources] +fuzzforge-common = { workspace = true } +fuzzforge-types = { workspace = true } diff --git a/fuzzforge-runner/pytest.ini b/fuzzforge-runner/pytest.ini new file mode 100644 index 0000000..c8c9c75 --- /dev/null +++ b/fuzzforge-runner/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode = auto +asyncio_default_fixture_loop_scope = function diff --git a/fuzzforge-runner/ruff.toml b/fuzzforge-runner/ruff.toml new file mode 100644 index 0000000..b9f8af9 --- /dev/null +++ b/fuzzforge-runner/ruff.toml @@ -0,0 +1 @@ +extend = "../ruff.toml" diff --git a/fuzzforge-runner/src/fuzzforge_runner/__init__.py b/fuzzforge-runner/src/fuzzforge_runner/__init__.py new file mode 100644 index 0000000..16f6ea6 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/__init__.py @@ -0,0 +1,9 @@ +"""FuzzForge Runner - Direct execution engine for FuzzForge OSS.""" + +from fuzzforge_runner.runner import Runner +from fuzzforge_runner.settings import Settings + +__all__ = [ + "Runner", + "Settings", +] diff --git a/fuzzforge-runner/src/fuzzforge_runner/__main__.py b/fuzzforge-runner/src/fuzzforge_runner/__main__.py new file mode 100644 index 0000000..36e4131 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/__main__.py @@ -0,0 +1,28 @@ +"""FuzzForge Runner CLI entry point.""" + +from fuzzforge_runner.runner import Runner +from fuzzforge_runner.settings import Settings + + +def main() -> None: + """Entry point for the FuzzForge Runner CLI. + + This is a minimal entry point that can be used for testing + and direct execution. The primary interface is via the MCP server. + + """ + import argparse + + parser = argparse.ArgumentParser(description="FuzzForge Runner") + parser.add_argument("--version", action="store_true", help="Print version and exit") + args = parser.parse_args() + + if args.version: + print("fuzzforge-runner 0.0.1") # noqa: T201 + return + + print("FuzzForge Runner - Use via MCP server or programmatically") # noqa: T201 + + +if __name__ == "__main__": + main() diff --git a/fuzzforge-runner/src/fuzzforge_runner/constants.py b/fuzzforge-runner/src/fuzzforge_runner/constants.py new file mode 100644 index 0000000..21d406a --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/constants.py @@ -0,0 +1,16 @@ +"""FuzzForge Runner constants.""" + +#: Default directory name for module input inside sandbox. +SANDBOX_INPUT_DIRECTORY: str = "/data/input" + +#: Default directory name for module output inside sandbox. +SANDBOX_OUTPUT_DIRECTORY: str = "/data/output" + +#: Default archive filename for results. +RESULTS_ARCHIVE_FILENAME: str = "results.tar.gz" + +#: Default configuration filename. +MODULE_CONFIG_FILENAME: str = "config.json" + +#: Module entrypoint script name. +MODULE_ENTRYPOINT: str = "module" diff --git a/fuzzforge-runner/src/fuzzforge_runner/exceptions.py b/fuzzforge-runner/src/fuzzforge_runner/exceptions.py new file mode 100644 index 0000000..ad98b2a --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/exceptions.py @@ -0,0 +1,27 @@ +"""FuzzForge Runner exceptions.""" + +from __future__ import annotations + + +class RunnerError(Exception): + """Base exception for all Runner errors.""" + + +class ModuleNotFoundError(RunnerError): + """Raised when a module cannot be found.""" + + +class ModuleExecutionError(RunnerError): + """Raised when module execution fails.""" + + +class WorkflowExecutionError(RunnerError): + """Raised when workflow execution fails.""" + + +class StorageError(RunnerError): + """Raised when storage operations fail.""" + + +class SandboxError(RunnerError): + """Raised when sandbox operations fail.""" diff --git a/fuzzforge-runner/src/fuzzforge_runner/executor.py b/fuzzforge-runner/src/fuzzforge_runner/executor.py new file mode 100644 index 0000000..652bc22 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/executor.py @@ -0,0 +1,635 @@ +"""FuzzForge Runner - Direct module execution engine. + +This module provides direct execution of FuzzForge modules without +requiring Temporal workflow orchestration. It's designed for local +development and OSS deployment scenarios. + +""" + +from __future__ import annotations + +import json +from io import BytesIO +from pathlib import Path, PurePath +from tarfile import TarFile, TarInfo +from tarfile import open as Archive # noqa: N812 +from tempfile import NamedTemporaryFile, TemporaryDirectory +from typing import TYPE_CHECKING, Any, cast + +from fuzzforge_common.sandboxes.engines.docker.configuration import DockerConfiguration +from fuzzforge_common.sandboxes.engines.podman.configuration import PodmanConfiguration +from fuzzforge_types.executions import FuzzForgeExecutionIdentifier + +from fuzzforge_runner.constants import ( + MODULE_ENTRYPOINT, + RESULTS_ARCHIVE_FILENAME, + SANDBOX_INPUT_DIRECTORY, + SANDBOX_OUTPUT_DIRECTORY, +) +from fuzzforge_runner.exceptions import ModuleExecutionError, SandboxError + +if TYPE_CHECKING: + from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine + from fuzzforge_runner.settings import EngineSettings, Settings + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """Get structlog logger instance. + + :returns: Configured structlog logger. + + """ + from structlog import get_logger # noqa: PLC0415 + + return cast("BoundLogger", get_logger()) + + +class ModuleExecutor: + """Direct executor for FuzzForge modules. + + Handles the complete lifecycle of module execution: + - Spawning isolated sandbox containers + - Pushing input assets and configuration + - Running the module + - Pulling output results + - Cleanup + + """ + + #: Full settings including engine and registry. + _settings: Settings + #: Engine settings for container operations. + _engine_settings: EngineSettings + + def __init__(self, settings: Settings) -> None: + """Initialize an instance of the class. + + :param settings: FuzzForge runner settings. + + """ + self._settings = settings + self._engine_settings = settings.engine + + def _get_engine_configuration(self) -> DockerConfiguration | PodmanConfiguration: + """Get the appropriate engine configuration. + + :returns: Engine configuration based on settings. + + Note: This is only used when socket mode is explicitly needed. + The default is now PodmanCLI with custom storage paths. + + """ + from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines + + # Ensure socket has proper scheme + socket = self._engine_settings.socket + if not socket.startswith(("unix://", "tcp://", "http://", "ssh://")): + socket = f"unix://{socket}" + + if self._engine_settings.type == "docker": + return DockerConfiguration( + kind=FuzzForgeSandboxEngines.DOCKER, + socket=socket, + ) + return PodmanConfiguration( + kind=FuzzForgeSandboxEngines.PODMAN, + socket=socket, + ) + + def _get_engine(self) -> AbstractFuzzForgeSandboxEngine: + """Get the container engine instance. + + Uses PodmanCLI with custom storage paths by default for Podman, + providing isolation from system Podman configuration and avoiding + issues with VS Code snap's XDG_DATA_HOME override. + + :returns: Configured container engine. + + """ + from fuzzforge_common.sandboxes.engines.podman import PodmanCLI + + # Use PodmanCLI with custom storage paths for Podman + if self._engine_settings.type == "podman": + return PodmanCLI( + graphroot=self._engine_settings.graphroot, + runroot=self._engine_settings.runroot, + ) + + # Fall back to socket-based engine for Docker + return self._get_engine_configuration().into_engine() + + def _check_image_exists(self, module_identifier: str) -> bool: + """Check if a module image exists locally. + + :param module_identifier: Name/identifier of the module image. + :returns: True if image exists, False otherwise. + + """ + engine = self._get_engine() + + # Try both common tags: latest and 0.0.1 + tags_to_check = ["latest", "0.0.1"] + + # Try both naming conventions: + # - localhost/fuzzforge-module-{name}:{tag} (standard convention) + # - localhost/{name}:{tag} (legacy/short form) + name_prefixes = [f"fuzzforge-module-{module_identifier}", module_identifier] + + for prefix in name_prefixes: + for tag in tags_to_check: + image_name = f"localhost/{prefix}:{tag}" + if engine.image_exists(image_name): + return True + + return False + + def _get_local_image_name(self, module_identifier: str) -> str: + """Get the full local image name for a module. + + :param module_identifier: Name/identifier of the module. + :returns: Full image name with localhost prefix. + + """ + engine = self._get_engine() + + # Check fuzzforge-module- prefix first (standard convention) + prefixed_name = f"localhost/fuzzforge-module-{module_identifier}:latest" + if engine.image_exists(prefixed_name): + return prefixed_name + + # Fall back to legacy short form + return f"localhost/{module_identifier}:latest" + + def _pull_module_image(self, module_identifier: str, registry_url: str = "ghcr.io/fuzzinglabs", tag: str = "latest") -> None: + """Pull a module image from the container registry. + + :param module_identifier: Name/identifier of the module to pull. + :param registry_url: Container registry URL. + :param tag: Image tag to pull. + :raises SandboxError: If pull fails. + + """ + logger = get_logger() + engine = self._get_engine() + + # Construct full image name + remote_image = f"{registry_url}/fuzzforge-module-{module_identifier}:{tag}" + local_image = f"localhost/{module_identifier}:{tag}" + + logger.info("pulling module image from registry", module=module_identifier, remote_image=remote_image) + + try: + # Pull the image using engine abstraction + engine.pull_image(remote_image, timeout=300) + + logger.info("module image pulled successfully", module=module_identifier) + + # Tag the image locally for consistency + engine.tag_image(remote_image, local_image) + + logger.debug("tagged image locally", local_image=local_image) + + except TimeoutError as exc: + message = f"Module image pull timed out after 5 minutes: {module_identifier}" + raise SandboxError(message) from exc + except Exception as exc: + message = ( + f"Failed to pull module image '{module_identifier}': {exc}\n" + f"Registry: {registry_url}\n" + f"Image: {remote_image}" + ) + raise SandboxError(message) from exc + + def _ensure_module_image(self, module_identifier: str, registry_url: str = "ghcr.io/fuzzinglabs", tag: str = "latest") -> None: + """Ensure module image exists, pulling it if necessary. + + :param module_identifier: Name/identifier of the module image. + :param registry_url: Container registry URL to pull from. + :param tag: Image tag to pull. + :raises SandboxError: If image check or pull fails. + + """ + logger = get_logger() + + if self._check_image_exists(module_identifier): + logger.debug("module image exists locally", module=module_identifier) + return + + logger.info( + "module image not found locally, pulling from registry", + module=module_identifier, + registry=registry_url, + info="This may take a moment on first run", + ) + self._pull_module_image(module_identifier, registry_url, tag) + + # Verify image now exists + if not self._check_image_exists(module_identifier): + message = ( + f"Module image '{module_identifier}' still not found after pull attempt.\n" + f"Tried to pull from: {registry_url}/fuzzforge-module-{module_identifier}:{tag}" + ) + raise SandboxError(message) + + def spawn_sandbox(self, module_identifier: str, input_volume: Path | None = None) -> str: + """Create and prepare a sandbox container for module execution. + + Automatically pulls the module image from registry if it doesn't exist locally. + + :param module_identifier: Name/identifier of the module image. + :param input_volume: Optional path to mount as /data/input in the container. + :returns: The sandbox container identifier. + :raises SandboxError: If sandbox creation fails. + + """ + logger = get_logger() + engine = self._get_engine() + + # Ensure module image exists (auto-pull if needed) + # Use registry settings from configuration + registry_url = self._settings.registry.url + tag = self._settings.registry.default_tag + self._ensure_module_image(module_identifier, registry_url, tag) + + logger.info("spawning sandbox", module=module_identifier) + try: + image = self._get_local_image_name(module_identifier) + + # Build volume mappings + volumes: dict[str, str] | None = None + if input_volume: + volumes = {str(input_volume): SANDBOX_INPUT_DIRECTORY} + + sandbox_id = engine.create_container(image=image, volumes=volumes) + logger.info("sandbox spawned", sandbox=sandbox_id, module=module_identifier) + return sandbox_id + + except TimeoutError as exc: + message = f"Container creation timed out for module {module_identifier}" + raise SandboxError(message) from exc + except Exception as exc: + message = f"Failed to spawn sandbox for module {module_identifier}" + raise SandboxError(message) from exc + + def prepare_input_directory( + self, + assets_path: Path, + configuration: dict[str, Any] | None = None, + ) -> Path: + """Prepare input directory with assets and configuration. + + Creates a temporary directory with input.json describing all resources. + This directory can be volume-mounted into the container. + + :param assets_path: Path to the assets (file or directory). + :param configuration: Optional module configuration dict. + :returns: Path to prepared input directory. + :raises SandboxError: If preparation fails. + + """ + logger = get_logger() + + logger.info("preparing input directory", assets=str(assets_path)) + + try: + # Create temporary directory - caller must clean it up after container finishes + from tempfile import mkdtemp + temp_path = Path(mkdtemp(prefix="fuzzforge-input-")) + + # Copy assets to temp directory + if assets_path.exists(): + if assets_path.is_file(): + # Check if it's a tar.gz archive that needs extraction + if assets_path.suffix == ".gz" or assets_path.name.endswith(".tar.gz"): + # Extract archive contents + import tarfile + with tarfile.open(assets_path, "r:gz") as tar: + tar.extractall(path=temp_path) + logger.debug("extracted tar.gz archive", archive=str(assets_path)) + else: + # Single file - copy it + import shutil + shutil.copy2(assets_path, temp_path / assets_path.name) + else: + # Directory - copy all files (including subdirectories) + import shutil + for item in assets_path.iterdir(): + if item.is_file(): + shutil.copy2(item, temp_path / item.name) + elif item.is_dir(): + shutil.copytree(item, temp_path / item.name) + + # Scan files and directories and build resource list + resources = [] + for item in temp_path.iterdir(): + if item.name == "input.json": + continue + if item.is_file(): + resources.append({ + "name": item.stem, + "description": f"Input file: {item.name}", + "kind": "unknown", + "path": f"/data/input/{item.name}", + }) + elif item.is_dir(): + resources.append({ + "name": item.name, + "description": f"Input directory: {item.name}", + "kind": "unknown", + "path": f"/data/input/{item.name}", + }) + + # Create input.json with settings and resources + input_data = { + "settings": configuration or {}, + "resources": resources, + } + input_json_path = temp_path / "input.json" + input_json_path.write_text(json.dumps(input_data, indent=2)) + + logger.debug("prepared input directory", resources=len(resources), path=str(temp_path)) + return temp_path + + except Exception as exc: + message = f"Failed to prepare input directory" + raise SandboxError(message) from exc + + def _push_config_to_sandbox(self, sandbox: str, configuration: dict[str, Any]) -> None: + """Write module configuration to sandbox as config.json. + + :param sandbox: The sandbox container identifier. + :param configuration: Configuration dictionary to write. + + """ + logger = get_logger() + engine = self._get_engine() + + logger.info("writing configuration to sandbox", sandbox=sandbox) + + with NamedTemporaryFile(mode="w", suffix=".json", delete=False) as config_file: + config_path = Path(config_file.name) + config_file.write(json.dumps(configuration, indent=2)) + + try: + engine.copy_to_container(sandbox, config_path, SANDBOX_INPUT_DIRECTORY) + except Exception as exc: + message = f"Failed to copy config.json: {exc}" + raise SandboxError(message) from exc + finally: + config_path.unlink() + + def run_module(self, sandbox: str) -> None: + """Start the sandbox and execute the module. + + :param sandbox: The sandbox container identifier. + :raises ModuleExecutionError: If module execution fails. + + """ + logger = get_logger() + engine = self._get_engine() + + logger.info("starting sandbox and running module", sandbox=sandbox) + try: + # The container runs its ENTRYPOINT (uv run module) when started + exit_code, stdout, stderr = engine.start_container_attached(sandbox, timeout=600) + + if exit_code != 0: + logger.error("module execution failed", sandbox=sandbox, stderr=stderr) + message = f"Module execution failed: {stderr}" + raise ModuleExecutionError(message) + logger.info("module execution completed", sandbox=sandbox) + + except TimeoutError as exc: + message = f"Module execution timed out after 10 minutes in sandbox {sandbox}" + raise ModuleExecutionError(message) from exc + except ModuleExecutionError: + raise + except Exception as exc: + message = f"Module execution failed in sandbox {sandbox}" + raise ModuleExecutionError(message) from exc + + def pull_results_from_sandbox(self, sandbox: str) -> Path: + """Pull the results archive from the sandbox. + + :param sandbox: The sandbox container identifier. + :returns: Path to the downloaded results archive (tar.gz file). + :raises SandboxError: If pull operation fails. + + """ + logger = get_logger() + engine = self._get_engine() + + logger.info("pulling results from sandbox", sandbox=sandbox) + try: + # Create temporary directory for results + from tempfile import mkdtemp + temp_dir = Path(mkdtemp(prefix="fuzzforge-results-")) + + # Copy entire output directory from container + try: + engine.copy_from_container(sandbox, SANDBOX_OUTPUT_DIRECTORY, temp_dir) + except Exception: + # If output directory doesn't exist, that's okay - module may not have produced results + logger.warning("no results found in sandbox", sandbox=sandbox) + + # Create tar archive from results directory + import tarfile + + archive_file = NamedTemporaryFile(delete=False, suffix=".tar.gz") + archive_path = Path(archive_file.name) + archive_file.close() + + with tarfile.open(archive_path, "w:gz") as tar: + # The output is extracted into a subdirectory named after the source + output_subdir = temp_dir / "output" + if output_subdir.exists(): + for item in output_subdir.iterdir(): + tar.add(item, arcname=item.name) + else: + for item in temp_dir.iterdir(): + tar.add(item, arcname=item.name) + + # Clean up temp directory + import shutil + shutil.rmtree(temp_dir, ignore_errors=True) + + logger.info("results pulled successfully", sandbox=sandbox, archive=str(archive_path)) + return archive_path + + except TimeoutError as exc: + message = f"Timeout pulling results from sandbox {sandbox}" + raise SandboxError(message) from exc + except Exception as exc: + message = f"Failed to pull results from sandbox {sandbox}" + raise SandboxError(message) from exc + + def terminate_sandbox(self, sandbox: str) -> None: + """Terminate and cleanup the sandbox container. + + :param sandbox: The sandbox container identifier. + + """ + logger = get_logger() + engine = self._get_engine() + + logger.info("terminating sandbox", sandbox=sandbox) + try: + engine.remove_container(sandbox, force=True) + logger.info("sandbox terminated", sandbox=sandbox) + except Exception as exc: + # Log but don't raise - cleanup should be best-effort + logger.warning("failed to terminate sandbox", sandbox=sandbox, error=str(exc)) + + async def execute( + self, + module_identifier: str, + assets_path: Path, + configuration: dict[str, Any] | None = None, + ) -> Path: + """Execute a module end-to-end. + + This is the main entry point that handles the complete execution flow: + 1. Spawn sandbox + 2. Push assets and configuration + 3. Run module + 4. Pull results + 5. Terminate sandbox + + :param module_identifier: Name/identifier of the module to execute. + :param assets_path: Path to the input assets archive. + :param configuration: Optional module configuration. + :returns: Path to the results archive. + :raises ModuleExecutionError: If any step fails. + + """ + logger = get_logger() + sandbox: str | None = None + input_dir: Path | None = None + + try: + # 1. Prepare input directory with assets + input_dir = self.prepare_input_directory(assets_path, configuration) + + # 2. Spawn sandbox with volume mount + sandbox = self.spawn_sandbox(module_identifier, input_volume=input_dir) + + # 3. Run module + self.run_module(sandbox) + + # 4. Pull results + results_path = self.pull_results_from_sandbox(sandbox) + + logger.info( + "module execution completed successfully", + module=module_identifier, + results=str(results_path), + ) + + return results_path + + finally: + # 5. Always cleanup + if sandbox: + self.terminate_sandbox(sandbox) + if input_dir and input_dir.exists(): + import shutil + shutil.rmtree(input_dir, ignore_errors=True) + + # ------------------------------------------------------------------------- + # Continuous/Background Execution Methods + # ------------------------------------------------------------------------- + + def start_module_continuous( + self, + module_identifier: str, + assets_path: Path, + configuration: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Start a module in continuous/background mode without waiting. + + Returns immediately with container info. Use read_module_output() to + get current status and stop_module_continuous() to stop. + + :param module_identifier: Name/identifier of the module to execute. + :param assets_path: Path to the input assets archive. + :param configuration: Optional module configuration. + :returns: Dict with container_id, input_dir for later cleanup. + + """ + logger = get_logger() + + # 1. Prepare input directory with assets + input_dir = self.prepare_input_directory(assets_path, configuration) + + # 2. Spawn sandbox with volume mount + sandbox = self.spawn_sandbox(module_identifier, input_volume=input_dir) + + # 3. Start container (non-blocking) + engine = self._get_engine() + engine.start_container(sandbox) + + logger.info( + "module started in continuous mode", + module=module_identifier, + container_id=sandbox, + ) + + return { + "container_id": sandbox, + "input_dir": str(input_dir), + "module": module_identifier, + } + + def read_module_output(self, container_id: str, output_file: str = "/data/output/stream.jsonl") -> str: + """Read output file from a running module container. + + :param container_id: The container identifier. + :param output_file: Path to output file inside container. + :returns: File contents as string. + + """ + engine = self._get_engine() + return engine.read_file_from_container(container_id, output_file) + + def get_module_status(self, container_id: str) -> str: + """Get the status of a running module container. + + :param container_id: The container identifier. + :returns: Container status (e.g., "running", "exited"). + + """ + engine = self._get_engine() + return engine.get_container_status(container_id) + + def stop_module_continuous(self, container_id: str, input_dir: str | None = None) -> Path: + """Stop a continuously running module and collect results. + + :param container_id: The container identifier. + :param input_dir: Optional input directory to cleanup. + :returns: Path to the results archive. + + """ + logger = get_logger() + engine = self._get_engine() + + try: + # 1. Stop the container gracefully + status = engine.get_container_status(container_id) + if status == "running": + engine.stop_container(container_id, timeout=10) + logger.info("stopped running container", container_id=container_id) + + # 2. Pull results + results_path = self.pull_results_from_sandbox(container_id) + + logger.info("collected results from continuous session", results=str(results_path)) + + return results_path + + finally: + # 3. Cleanup + self.terminate_sandbox(container_id) + if input_dir: + import shutil + shutil.rmtree(input_dir, ignore_errors=True) diff --git a/fuzzforge-runner/src/fuzzforge_runner/orchestrator.py b/fuzzforge-runner/src/fuzzforge_runner/orchestrator.py new file mode 100644 index 0000000..a541457 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/orchestrator.py @@ -0,0 +1,360 @@ +"""FuzzForge Runner - Workflow orchestration without Temporal. + +This module provides simplified workflow orchestration for sequential +module execution without requiring Temporal infrastructure. + +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast +from uuid import uuid4 + +from fuzzforge_types.executions import FuzzForgeExecutionIdentifier + +from fuzzforge_runner.exceptions import WorkflowExecutionError +from fuzzforge_runner.executor import ModuleExecutor + +if TYPE_CHECKING: + from fuzzforge_runner.settings import Settings + from fuzzforge_runner.storage import LocalStorage + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """Get structlog logger instance. + + :returns: Configured structlog logger. + + """ + from structlog import get_logger # noqa: PLC0415 + + return cast("BoundLogger", get_logger()) + + +@dataclass +class WorkflowStep: + """Represents a single step in a workflow.""" + + #: Module identifier to execute. + module_identifier: str + + #: Optional configuration for the module. + configuration: dict[str, Any] | None = None + + #: Step name/label for logging. + name: str | None = None + + +@dataclass +class WorkflowDefinition: + """Defines a workflow as a sequence of module executions.""" + + #: Workflow name. + name: str + + #: Ordered list of steps to execute. + steps: list[WorkflowStep] = field(default_factory=list) + + #: Optional workflow description. + description: str | None = None + + +@dataclass +class StepResult: + """Result of a single workflow step execution.""" + + #: Step index (0-based). + step_index: int + + #: Module that was executed. + module_identifier: str + + #: Path to the results archive. + results_path: Path + + #: Execution identifier. + execution_id: str + + #: Execution start time. + started_at: datetime + + #: Execution end time. + completed_at: datetime + + #: Whether execution was successful. + success: bool = True + + #: Error message if failed. + error: str | None = None + + +@dataclass +class WorkflowResult: + """Result of a complete workflow execution.""" + + #: Workflow execution identifier. + execution_id: str + + #: Workflow name. + name: str + + #: Results for each step. + steps: list[StepResult] = field(default_factory=list) + + #: Overall success status. + success: bool = True + + #: Final results path (from last step). + final_results_path: Path | None = None + + +class WorkflowOrchestrator: + """Orchestrates sequential workflow execution. + + Executes workflow steps sequentially, passing output from each + module as input to the next. No Temporal required. + + """ + + #: Module executor instance. + _executor: ModuleExecutor + + #: Storage backend. + _storage: LocalStorage + + def __init__(self, executor: ModuleExecutor, storage: LocalStorage) -> None: + """Initialize an instance of the class. + + :param executor: Module executor for running modules. + :param storage: Storage backend for managing assets. + + """ + self._executor = executor + self._storage = storage + + def _generate_execution_id(self) -> str: + """Generate a unique execution identifier. + + :returns: UUID string for execution tracking. + + """ + return str(uuid4()) + + async def execute_workflow( + self, + workflow: WorkflowDefinition, + project_path: Path, + initial_assets_path: Path | None = None, + ) -> WorkflowResult: + """Execute a workflow as a sequence of module executions. + + Each step receives the output of the previous step as input. + The first step receives the initial assets. + + :param workflow: Workflow definition with steps to execute. + :param project_path: Path to the project directory. + :param initial_assets_path: Path to initial assets (optional). + :returns: Workflow execution result. + :raises WorkflowExecutionError: If workflow execution fails. + + """ + logger = get_logger() + workflow_id = self._generate_execution_id() + + logger.info( + "starting workflow execution", + workflow=workflow.name, + execution_id=workflow_id, + steps=len(workflow.steps), + ) + + result = WorkflowResult( + execution_id=workflow_id, + name=workflow.name, + ) + + if not workflow.steps: + logger.warning("workflow has no steps", workflow=workflow.name) + return result + + # Track current assets path - starts with initial assets, then uses previous step output + current_assets: Path | None = initial_assets_path + + # If no initial assets, try to get from project + if current_assets is None: + current_assets = self._storage.get_project_assets_path(project_path) + + try: + for step_index, step in enumerate(workflow.steps): + step_name = step.name or f"step-{step_index}" + step_execution_id = self._generate_execution_id() + + logger.info( + "executing workflow step", + workflow=workflow.name, + step=step_name, + step_index=step_index, + module=step.module_identifier, + execution_id=step_execution_id, + ) + + started_at = datetime.now(UTC) + + try: + # Ensure we have assets for this step + if current_assets is None or not current_assets.exists(): + if step_index == 0: + # First step with no assets - create empty archive + current_assets = self._storage.create_empty_assets_archive(project_path) + else: + message = f"No assets available for step {step_index}" + raise WorkflowExecutionError(message) + + # Execute the module + results_path = await self._executor.execute( + module_identifier=step.module_identifier, + assets_path=current_assets, + configuration=step.configuration, + ) + + completed_at = datetime.now(UTC) + + # Store results to persistent storage + stored_path = self._storage.store_execution_results( + project_path=project_path, + workflow_id=workflow_id, + step_index=step_index, + execution_id=step_execution_id, + results_path=results_path, + ) + + # Clean up temporary results archive after storing + try: + if results_path.exists() and results_path != stored_path: + results_path.unlink() + except Exception as cleanup_exc: + logger.warning("failed to clean up temporary results", path=str(results_path), error=str(cleanup_exc)) + + # Record step result with stored path + step_result = StepResult( + step_index=step_index, + module_identifier=step.module_identifier, + results_path=stored_path, + execution_id=step_execution_id, + started_at=started_at, + completed_at=completed_at, + success=True, + ) + result.steps.append(step_result) + + # Next step uses this step's output + current_assets = stored_path + + logger.info( + "workflow step completed", + step=step_name, + step_index=step_index, + duration_seconds=(completed_at - started_at).total_seconds(), + ) + + except Exception as exc: + completed_at = datetime.now(UTC) + error_msg = str(exc) + + step_result = StepResult( + step_index=step_index, + module_identifier=step.module_identifier, + results_path=Path(), + execution_id=step_execution_id, + started_at=started_at, + completed_at=completed_at, + success=False, + error=error_msg, + ) + result.steps.append(step_result) + result.success = False + + logger.error( + "workflow step failed", + step=step_name, + step_index=step_index, + error=error_msg, + ) + + # Stop workflow on failure + break + + # Set final results path + if result.steps and result.steps[-1].success: + result.final_results_path = result.steps[-1].results_path + + logger.info( + "workflow execution completed", + workflow=workflow.name, + execution_id=workflow_id, + success=result.success, + completed_steps=len([s for s in result.steps if s.success]), + total_steps=len(workflow.steps), + ) + + return result + + except Exception as exc: + message = f"Workflow execution failed: {exc}" + logger.exception("workflow execution error", workflow=workflow.name) + raise WorkflowExecutionError(message) from exc + + async def execute_single_module( + self, + module_identifier: str, + project_path: Path, + assets_path: Path | None = None, + configuration: dict[str, Any] | None = None, + ) -> StepResult: + """Execute a single module (convenience method). + + This is a simplified interface for executing a single module + outside of a workflow context. + + :param module_identifier: Module to execute. + :param project_path: Project directory path. + :param assets_path: Optional path to input assets. + :param configuration: Optional module configuration. + :returns: Execution result. + + """ + workflow = WorkflowDefinition( + name=f"single-{module_identifier}", + steps=[ + WorkflowStep( + module_identifier=module_identifier, + configuration=configuration, + name="main", + ) + ], + ) + + result = await self.execute_workflow( + workflow=workflow, + project_path=project_path, + initial_assets_path=assets_path, + ) + + if result.steps: + return result.steps[0] + + # Should not happen, but handle gracefully + return StepResult( + step_index=0, + module_identifier=module_identifier, + results_path=Path(), + execution_id=result.execution_id, + started_at=datetime.now(UTC), + completed_at=datetime.now(UTC), + success=False, + error="No step results produced", + ) diff --git a/fuzzforge-runner/src/fuzzforge_runner/runner.py b/fuzzforge-runner/src/fuzzforge_runner/runner.py new file mode 100644 index 0000000..1ef0bf8 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/runner.py @@ -0,0 +1,378 @@ +"""FuzzForge Runner - Main runner interface. + +This module provides the high-level interface for FuzzForge OSS, +coordinating module execution, workflow orchestration, and storage. + +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +from fuzzforge_runner.executor import ModuleExecutor +from fuzzforge_runner.orchestrator import ( + StepResult, + WorkflowDefinition, + WorkflowOrchestrator, + WorkflowResult, + WorkflowStep, +) +from fuzzforge_runner.settings import Settings +from fuzzforge_runner.storage import LocalStorage + +if TYPE_CHECKING: + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """Get structlog logger instance. + + :returns: Configured structlog logger. + + """ + from structlog import get_logger # noqa: PLC0415 + + return cast("BoundLogger", get_logger()) + + +@dataclass +class ModuleInfo: + """Information about an available module.""" + + #: Module identifier/name. + identifier: str + + #: Module description. + description: str | None = None + + #: Module version. + version: str | None = None + + #: Whether module image exists locally. + available: bool = True + + +class Runner: + """Main FuzzForge Runner interface. + + Provides a unified interface for: + - Module discovery and execution + - Workflow orchestration + - Project and asset management + + This is the primary entry point for OSS users and the MCP server. + + """ + + #: Runner settings. + _settings: Settings + + #: Module executor. + _executor: ModuleExecutor + + #: Local storage backend. + _storage: LocalStorage + + #: Workflow orchestrator. + _orchestrator: WorkflowOrchestrator + + def __init__(self, settings: Settings | None = None) -> None: + """Initialize an instance of the class. + + :param settings: Runner settings. If None, loads from environment. + + """ + self._settings = settings or Settings() + self._executor = ModuleExecutor(self._settings) + self._storage = LocalStorage(self._settings.storage.path) + self._orchestrator = WorkflowOrchestrator(self._executor, self._storage) + + @property + def settings(self) -> Settings: + """Get runner settings. + + :returns: Current settings instance. + + """ + return self._settings + + @property + def storage(self) -> LocalStorage: + """Get storage backend. + + :returns: Storage instance. + + """ + return self._storage + + # ------------------------------------------------------------------------- + # Project Management + # ------------------------------------------------------------------------- + + def init_project(self, project_path: Path) -> Path: + """Initialize a new project. + + Creates necessary storage directories for a project. + + :param project_path: Path to the project directory. + :returns: Path to the project storage directory. + + """ + logger = get_logger() + logger.info("initializing project", path=str(project_path)) + return self._storage.init_project(project_path) + + def set_project_assets(self, project_path: Path, assets_path: Path) -> Path: + """Set initial assets for a project. + + :param project_path: Path to the project directory. + :param assets_path: Path to assets (file or directory). + :returns: Path to stored assets. + + """ + logger = get_logger() + logger.info("setting project assets", project=str(project_path), assets=str(assets_path)) + return self._storage.store_assets(project_path, assets_path) + + # ------------------------------------------------------------------------- + # Module Discovery + # ------------------------------------------------------------------------- + + def list_modules(self) -> list[ModuleInfo]: + """List available modules. + + Discovers modules from the configured modules directory. + + :returns: List of available modules. + + """ + logger = get_logger() + modules: list[ModuleInfo] = [] + + modules_path = self._settings.modules_path + if not modules_path.exists(): + logger.warning("modules directory not found", path=str(modules_path)) + return modules + + # Look for module directories (each should have a Dockerfile or be a built image) + for item in modules_path.iterdir(): + if item.is_dir(): + # Check for module markers + has_dockerfile = (item / "Dockerfile").exists() + has_pyproject = (item / "pyproject.toml").exists() + + if has_dockerfile or has_pyproject: + modules.append( + ModuleInfo( + identifier=item.name, + available=has_dockerfile, + ) + ) + + logger.info("discovered modules", count=len(modules)) + return modules + + def list_module_images(self, filter_prefix: str = "localhost/") -> list[ModuleInfo]: + """List available module images from the container engine. + + Uses the container engine API to discover built module images. + + :param filter_prefix: Prefix to filter images (default: "localhost/"). + :returns: List of available module images. + + """ + logger = get_logger() + modules: list[ModuleInfo] = [] + seen: set[str] = set() + + # Infrastructure images to skip + skip_images = {"fuzzforge-modules-sdk", "fuzzforge-runner", "fuzzforge-api"} + + engine = self._executor._get_engine() + images = engine.list_images(filter_prefix=filter_prefix) + + for image in images: + # Only include :latest images + if image.tag != "latest": + continue + + # Extract module name from repository + full_name = image.repository.split("/")[-1] + + # Skip infrastructure images + if full_name in skip_images: + continue + + # Extract clean module name (remove fuzzforge-module- prefix if present) + if full_name.startswith("fuzzforge-module-"): + module_name = full_name.replace("fuzzforge-module-", "") + else: + module_name = full_name + + # Skip UUID-like names (temporary/broken containers) + if module_name.count("-") >= 4 and len(module_name) > 30: + continue + + # Add unique modules + if module_name not in seen: + seen.add(module_name) + modules.append( + ModuleInfo( + identifier=module_name, + description=None, + version=image.tag, + available=True, + ) + ) + + logger.info("listed module images", count=len(modules)) + return modules + + def get_module_info(self, module_identifier: str) -> ModuleInfo | None: + """Get information about a specific module. + + :param module_identifier: Module identifier to look up. + :returns: Module info, or None if not found. + + """ + modules = self.list_modules() + for module in modules: + if module.identifier == module_identifier: + return module + return None + + # ------------------------------------------------------------------------- + # Module Execution + # ------------------------------------------------------------------------- + + async def execute_module( + self, + module_identifier: str, + project_path: Path, + configuration: dict[str, Any] | None = None, + assets_path: Path | None = None, + ) -> StepResult: + """Execute a single module. + + :param module_identifier: Module to execute. + :param project_path: Path to the project directory. + :param configuration: Optional module configuration. + :param assets_path: Optional path to input assets. + :returns: Execution result. + + """ + logger = get_logger() + logger.info( + "executing module", + module=module_identifier, + project=str(project_path), + ) + + return await self._orchestrator.execute_single_module( + module_identifier=module_identifier, + project_path=project_path, + assets_path=assets_path, + configuration=configuration, + ) + + # ------------------------------------------------------------------------- + # Workflow Execution + # ------------------------------------------------------------------------- + + async def execute_workflow( + self, + workflow: WorkflowDefinition, + project_path: Path, + initial_assets_path: Path | None = None, + ) -> WorkflowResult: + """Execute a workflow. + + :param workflow: Workflow definition with steps. + :param project_path: Path to the project directory. + :param initial_assets_path: Optional path to initial assets. + :returns: Workflow execution result. + + """ + logger = get_logger() + logger.info( + "executing workflow", + workflow=workflow.name, + project=str(project_path), + steps=len(workflow.steps), + ) + + return await self._orchestrator.execute_workflow( + workflow=workflow, + project_path=project_path, + initial_assets_path=initial_assets_path, + ) + + def create_workflow( + self, + name: str, + steps: list[tuple[str, dict[str, Any] | None]], + description: str | None = None, + ) -> WorkflowDefinition: + """Create a workflow definition. + + Convenience method for creating workflows programmatically. + + :param name: Workflow name. + :param steps: List of (module_identifier, configuration) tuples. + :param description: Optional workflow description. + :returns: Workflow definition. + + """ + workflow_steps = [ + WorkflowStep( + module_identifier=module_id, + configuration=config, + name=f"step-{i}", + ) + for i, (module_id, config) in enumerate(steps) + ] + + return WorkflowDefinition( + name=name, + steps=workflow_steps, + description=description, + ) + + # ------------------------------------------------------------------------- + # Results Management + # ------------------------------------------------------------------------- + + def get_execution_results( + self, + project_path: Path, + execution_id: str, + ) -> Path | None: + """Get results for an execution. + + :param project_path: Path to the project directory. + :param execution_id: Execution ID. + :returns: Path to results archive, or None if not found. + + """ + return self._storage.get_execution_results(project_path, execution_id) + + def list_executions(self, project_path: Path) -> list[str]: + """List all executions for a project. + + :param project_path: Path to the project directory. + :returns: List of execution IDs. + + """ + return self._storage.list_executions(project_path) + + def extract_results(self, results_path: Path, destination: Path) -> Path: + """Extract results archive to a directory. + + :param results_path: Path to results archive. + :param destination: Destination directory. + :returns: Path to extracted directory. + + """ + return self._storage.extract_results(results_path, destination) diff --git a/fuzzforge-runner/src/fuzzforge_runner/settings.py b/fuzzforge-runner/src/fuzzforge_runner/settings.py new file mode 100644 index 0000000..4e4d630 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/settings.py @@ -0,0 +1,114 @@ +"""FuzzForge Runner settings configuration.""" + +from __future__ import annotations + +from enum import StrEnum +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class EngineType(StrEnum): + """Supported container engine types.""" + + DOCKER = "docker" + PODMAN = "podman" + + +class EngineSettings(BaseModel): + """Container engine configuration.""" + + #: Type of container engine to use. + type: EngineType = EngineType.PODMAN + + #: Path to the container engine socket (only used as fallback). + socket: str = Field(default="") + + #: Custom graph root for container storage (isolated from system). + #: When set, uses CLI mode instead of socket for better portability. + graphroot: Path = Field(default=Path.home() / ".fuzzforge" / "containers" / "storage") + + #: Custom run root for container runtime state. + runroot: Path = Field(default=Path.home() / ".fuzzforge" / "containers" / "run") + + +class StorageSettings(BaseModel): + """Storage configuration for local or S3 storage.""" + + #: Storage backend type. + type: Literal["local", "s3"] = "local" + + #: Base path for local storage (used when type is "local"). + path: Path = Field(default=Path.home() / ".fuzzforge" / "storage") + + #: S3 endpoint URL (used when type is "s3"). + s3_endpoint: str | None = None + + #: S3 access key (used when type is "s3"). + s3_access_key: str | None = None + + #: S3 secret key (used when type is "s3"). + s3_secret_key: str | None = None + + +class ProjectSettings(BaseModel): + """Project configuration.""" + + #: Default path for FuzzForge projects. + default_path: Path = Field(default=Path.home() / ".fuzzforge" / "projects") + + +class RegistrySettings(BaseModel): + """Container registry configuration for module images.""" + + #: Registry URL for pulling module images. + url: str = Field(default="ghcr.io/fuzzinglabs") + + #: Default tag to use when pulling images. + default_tag: str = Field(default="latest") + + #: Registry username for authentication (optional). + username: str | None = None + + #: Registry password/token for authentication (optional). + password: str | None = None + + +class Settings(BaseSettings): + """FuzzForge Runner settings. + + Settings can be configured via environment variables with the prefix + ``FUZZFORGE_``. Nested settings use underscore as delimiter. + + Example: + ``FUZZFORGE_ENGINE_TYPE=docker`` + ``FUZZFORGE_STORAGE_PATH=/data/fuzzforge`` + ``FUZZFORGE_MODULES_PATH=/path/to/modules`` + + """ + + model_config = SettingsConfigDict( + case_sensitive=False, + env_nested_delimiter="__", + env_prefix="FUZZFORGE_", + ) + + #: Container engine settings. + engine: EngineSettings = Field(default_factory=EngineSettings) + + #: Storage settings. + storage: StorageSettings = Field(default_factory=StorageSettings) + + #: Project settings. + project: ProjectSettings = Field(default_factory=ProjectSettings) + + #: Container registry settings. + registry: RegistrySettings = Field(default_factory=RegistrySettings) + + #: Path to modules directory (for development/local builds). + modules_path: Path = Field(default=Path.home() / ".fuzzforge" / "modules") + + #: Enable debug logging. + debug: bool = False diff --git a/fuzzforge-runner/src/fuzzforge_runner/storage.py b/fuzzforge-runner/src/fuzzforge_runner/storage.py new file mode 100644 index 0000000..e538361 --- /dev/null +++ b/fuzzforge-runner/src/fuzzforge_runner/storage.py @@ -0,0 +1,359 @@ +"""FuzzForge Runner - Local filesystem storage. + +This module provides local filesystem storage as an alternative to S3, +enabling zero-configuration operation for OSS deployments. + +""" + +from __future__ import annotations + +import shutil +from pathlib import Path, PurePath +from tarfile import open as Archive # noqa: N812 +from tempfile import NamedTemporaryFile, TemporaryDirectory +from typing import TYPE_CHECKING, cast + +from fuzzforge_runner.constants import RESULTS_ARCHIVE_FILENAME +from fuzzforge_runner.exceptions import StorageError + +if TYPE_CHECKING: + from structlog.stdlib import BoundLogger + + +def get_logger() -> BoundLogger: + """Get structlog logger instance. + + :returns: Configured structlog logger. + + """ + from structlog import get_logger # noqa: PLC0415 + + return cast("BoundLogger", get_logger()) + + +class LocalStorage: + """Local filesystem storage backend. + + Provides S3-like operations using local filesystem, enabling + FuzzForge operation without external storage infrastructure. + + Directory structure: + {base_path}/ + projects/ + {project_id}/ + assets/ # Initial project assets + runs/ + {execution_id}/ + results.tar.gz + {workflow_id}/ + modules/ + step-0-{exec_id}/ + results.tar.gz + + """ + + #: Base path for all storage operations. + _base_path: Path + + def __init__(self, base_path: Path) -> None: + """Initialize an instance of the class. + + :param base_path: Root directory for storage. + + """ + self._base_path = base_path + self._ensure_base_path() + + def _ensure_base_path(self) -> None: + """Ensure the base storage directory exists.""" + self._base_path.mkdir(parents=True, exist_ok=True) + + def _get_project_path(self, project_path: Path) -> Path: + """Get the storage path for a project. + + :param project_path: Original project path (used as identifier). + :returns: Storage path for the project. + + """ + # Use project path name as identifier + project_id = project_path.name + return self._base_path / "projects" / project_id + + def init_project(self, project_path: Path) -> Path: + """Initialize storage for a new project. + + :param project_path: Path to the project directory. + :returns: Path to the project storage directory. + + """ + logger = get_logger() + storage_path = self._get_project_path(project_path) + + # Create directory structure + (storage_path / "assets").mkdir(parents=True, exist_ok=True) + (storage_path / "runs").mkdir(parents=True, exist_ok=True) + + logger.info("initialized project storage", project=project_path.name, storage=str(storage_path)) + + return storage_path + + def get_project_assets_path(self, project_path: Path) -> Path | None: + """Get the path to project assets archive. + + :param project_path: Path to the project directory. + :returns: Path to assets archive, or None if not found. + + """ + storage_path = self._get_project_path(project_path) + assets_dir = storage_path / "assets" + + # Look for assets archive + archive_path = assets_dir / "assets.tar.gz" + if archive_path.exists(): + return archive_path + + # Check if there are any files in assets directory + if assets_dir.exists() and any(assets_dir.iterdir()): + # Create archive from directory contents + return self._create_archive_from_directory(assets_dir) + + return None + + def _create_archive_from_directory(self, directory: Path) -> Path: + """Create a tar.gz archive from a directory's contents. + + :param directory: Directory to archive. + :returns: Path to the created archive. + + """ + archive_path = directory.parent / f"{directory.name}.tar.gz" + + with Archive(archive_path, "w:gz") as tar: + for item in directory.iterdir(): + tar.add(item, arcname=item.name) + + return archive_path + + def create_empty_assets_archive(self, project_path: Path) -> Path: + """Create an empty assets archive for a project. + + :param project_path: Path to the project directory. + :returns: Path to the empty archive. + + """ + storage_path = self._get_project_path(project_path) + assets_dir = storage_path / "assets" + assets_dir.mkdir(parents=True, exist_ok=True) + + archive_path = assets_dir / "assets.tar.gz" + + # Create empty archive + with Archive(archive_path, "w:gz") as tar: + pass # Empty archive + + return archive_path + + def store_assets(self, project_path: Path, assets_path: Path) -> Path: + """Store project assets from a local path. + + :param project_path: Path to the project directory. + :param assets_path: Source path (file or directory) to store. + :returns: Path to the stored assets. + :raises StorageError: If storage operation fails. + + """ + logger = get_logger() + storage_path = self._get_project_path(project_path) + assets_dir = storage_path / "assets" + assets_dir.mkdir(parents=True, exist_ok=True) + + try: + if assets_path.is_file(): + # Copy archive directly + dest_path = assets_dir / "assets.tar.gz" + shutil.copy2(assets_path, dest_path) + else: + # Create archive from directory + dest_path = assets_dir / "assets.tar.gz" + with Archive(dest_path, "w:gz") as tar: + for item in assets_path.iterdir(): + tar.add(item, arcname=item.name) + + logger.info("stored project assets", project=project_path.name, path=str(dest_path)) + return dest_path + + except Exception as exc: + message = f"Failed to store assets: {exc}" + raise StorageError(message) from exc + + def store_execution_results( + self, + project_path: Path, + workflow_id: str | None, + step_index: int, + execution_id: str, + results_path: Path, + ) -> Path: + """Store execution results. + + :param project_path: Path to the project directory. + :param workflow_id: Workflow execution ID (None for standalone). + :param step_index: Step index in workflow. + :param execution_id: Module execution ID. + :param results_path: Path to results archive to store. + :returns: Path to the stored results. + :raises StorageError: If storage operation fails. + + """ + logger = get_logger() + storage_path = self._get_project_path(project_path) + + try: + if workflow_id: + # Part of workflow + dest_dir = storage_path / "runs" / workflow_id / "modules" / f"step-{step_index}-{execution_id}" + else: + # Standalone execution + dest_dir = storage_path / "runs" / execution_id + + dest_dir.mkdir(parents=True, exist_ok=True) + dest_path = dest_dir / RESULTS_ARCHIVE_FILENAME + + shutil.copy2(results_path, dest_path) + + logger.info( + "stored execution results", + execution_id=execution_id, + path=str(dest_path), + ) + + return dest_path + + except Exception as exc: + message = f"Failed to store results: {exc}" + raise StorageError(message) from exc + + def get_execution_results( + self, + project_path: Path, + execution_id: str, + workflow_id: str | None = None, + step_index: int | None = None, + ) -> Path | None: + """Retrieve execution results. + + :param project_path: Path to the project directory. + :param execution_id: Module execution ID. + :param workflow_id: Workflow execution ID (None for standalone). + :param step_index: Step index in workflow. + :returns: Path to results archive, or None if not found. + + """ + storage_path = self._get_project_path(project_path) + + if workflow_id and step_index is not None: + # Direct workflow path lookup + results_path = ( + storage_path / "runs" / workflow_id / "modules" / f"step-{step_index}-{execution_id}" / RESULTS_ARCHIVE_FILENAME + ) + if results_path.exists(): + return results_path + + # Try standalone path + results_path = storage_path / "runs" / execution_id / RESULTS_ARCHIVE_FILENAME + if results_path.exists(): + return results_path + + # Search for execution_id in all workflow runs + runs_dir = storage_path / "runs" + if runs_dir.exists(): + for workflow_dir in runs_dir.iterdir(): + if not workflow_dir.is_dir(): + continue + + # Check if this is a workflow directory (has 'modules' subdirectory) + modules_dir = workflow_dir / "modules" + if modules_dir.exists() and modules_dir.is_dir(): + # Search for step directories containing this execution_id + for step_dir in modules_dir.iterdir(): + if step_dir.is_dir() and execution_id in step_dir.name: + results_path = step_dir / RESULTS_ARCHIVE_FILENAME + if results_path.exists(): + return results_path + + return None + + def list_executions(self, project_path: Path) -> list[str]: + """List all execution IDs for a project. + + :param project_path: Path to the project directory. + :returns: List of execution IDs. + + """ + storage_path = self._get_project_path(project_path) + runs_dir = storage_path / "runs" + + if not runs_dir.exists(): + return [] + + return [d.name for d in runs_dir.iterdir() if d.is_dir()] + + def delete_execution(self, project_path: Path, execution_id: str) -> bool: + """Delete an execution and its results. + + :param project_path: Path to the project directory. + :param execution_id: Execution ID to delete. + :returns: True if deleted, False if not found. + + """ + logger = get_logger() + storage_path = self._get_project_path(project_path) + exec_path = storage_path / "runs" / execution_id + + if exec_path.exists(): + shutil.rmtree(exec_path) + logger.info("deleted execution", execution_id=execution_id) + return True + + return False + + def delete_project(self, project_path: Path) -> bool: + """Delete all storage for a project. + + :param project_path: Path to the project directory. + :returns: True if deleted, False if not found. + + """ + logger = get_logger() + storage_path = self._get_project_path(project_path) + + if storage_path.exists(): + shutil.rmtree(storage_path) + logger.info("deleted project storage", project=project_path.name) + return True + + return False + + def extract_results(self, results_path: Path, destination: Path) -> Path: + """Extract a results archive to a destination directory. + + :param results_path: Path to the results archive. + :param destination: Directory to extract to. + :returns: Path to extracted directory. + :raises StorageError: If extraction fails. + + """ + logger = get_logger() + + try: + destination.mkdir(parents=True, exist_ok=True) + + with Archive(results_path, "r:gz") as tar: + tar.extractall(path=destination) + + logger.info("extracted results", source=str(results_path), destination=str(destination)) + return destination + + except Exception as exc: + message = f"Failed to extract results: {exc}" + raise StorageError(message) from exc diff --git a/fuzzforge-runner/tests/__init__.py b/fuzzforge-runner/tests/__init__.py new file mode 100644 index 0000000..85ff61d --- /dev/null +++ b/fuzzforge-runner/tests/__init__.py @@ -0,0 +1 @@ +"""Tests package for fuzzforge-runner.""" diff --git a/fuzzforge-runner/tests/conftest.py b/fuzzforge-runner/tests/conftest.py new file mode 100644 index 0000000..29bc724 --- /dev/null +++ b/fuzzforge-runner/tests/conftest.py @@ -0,0 +1,3 @@ +"""Pytest configuration for fuzzforge-runner tests.""" + +import pytest diff --git a/fuzzforge-tests/Makefile b/fuzzforge-tests/Makefile new file mode 100644 index 0000000..5218735 --- /dev/null +++ b/fuzzforge-tests/Makefile @@ -0,0 +1,30 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +SOURCES=./src + +.PHONY: clean format mypy ruff version + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.pytest_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) + +mypy: + uv run mypy $(SOURCES) + +ruff: + uv run ruff check --fix $(SOURCES) + +version: + @echo '$(PACKAGE)@$(VERSION)' diff --git a/fuzzforge-tests/README.md b/fuzzforge-tests/README.md new file mode 100644 index 0000000..2b71651 --- /dev/null +++ b/fuzzforge-tests/README.md @@ -0,0 +1,3 @@ +# fuzzforge-tests + +Common test utilities and fixtures for FuzzForge packages. diff --git a/fuzzforge-tests/mypy.ini b/fuzzforge-tests/mypy.ini new file mode 100644 index 0000000..bb7847e --- /dev/null +++ b/fuzzforge-tests/mypy.ini @@ -0,0 +1,12 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True + +[mypy-boto3.*] +ignore_missing_imports = True + +[mypy-testcontainers.*] +ignore_missing_imports = True diff --git a/fuzzforge-tests/pyproject.toml b/fuzzforge-tests/pyproject.toml new file mode 100644 index 0000000..57a1ea5 --- /dev/null +++ b/fuzzforge-tests/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "fuzzforge-tests" +version = "0.0.1" +description = "Common test utilities and fixtures for FuzzForge packages." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "boto3==1.42.8", + "podman==5.6.0", + "pytest==9.0.2", + "fuzzforge-types==0.0.1", + "testcontainers[minio]==4.13.3", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] + +[tool.uv.sources] +fuzzforge-types = { workspace = true } diff --git a/fuzzforge-tests/ruff.toml b/fuzzforge-tests/ruff.toml new file mode 100644 index 0000000..e3f2063 --- /dev/null +++ b/fuzzforge-tests/ruff.toml @@ -0,0 +1,16 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements + "PLR0913", # allowing functions with many arguments in tests (required for fixtures) + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-tests/src/fuzzforge_tests/__init__.py b/fuzzforge-tests/src/fuzzforge_tests/__init__.py new file mode 100644 index 0000000..27fa30a --- /dev/null +++ b/fuzzforge-tests/src/fuzzforge_tests/__init__.py @@ -0,0 +1,9 @@ +"""Common test utilities and fixtures for FuzzForge packages. + +This package provides shared test utilities, fixtures, and helpers that can be +reused across multiple FuzzForge packages to reduce code duplication and ensure +consistency in testing approaches. + +""" + +__all__ = [] diff --git a/fuzzforge-tests/src/fuzzforge_tests/conftest.py b/fuzzforge-tests/src/fuzzforge_tests/conftest.py new file mode 100644 index 0000000..58d5096 --- /dev/null +++ b/fuzzforge-tests/src/fuzzforge_tests/conftest.py @@ -0,0 +1,22 @@ +"""Pytest configuration for shared fixtures. + +This conftest.py makes fixtures available to any test that imports from +fuzzforge_tests. Test packages should add 'pytest_plugins = ["fuzzforge_tests.fixtures"]' +to their conftest.py to use these shared fixtures. + +""" + +# Import fixtures to make them available +from fuzzforge_tests.fixtures import ( + minio_container, + random_module_execution_identifier, + random_project_identifier, + storage_configuration, +) + +__all__ = [ + "minio_container", + "random_module_execution_identifier", + "random_project_identifier", + "storage_configuration", +] diff --git a/fuzzforge-tests/src/fuzzforge_tests/fixtures.py b/fuzzforge-tests/src/fuzzforge_tests/fixtures.py new file mode 100644 index 0000000..c9ea0d5 --- /dev/null +++ b/fuzzforge-tests/src/fuzzforge_tests/fixtures.py @@ -0,0 +1,301 @@ +"""Common test fixtures for FuzzForge packages. + +Provides reusable fixtures for generating random identifiers and other +common test utilities shared across multiple FuzzForge packages. + +""" + +import random +import string +from os import environ +from typing import TYPE_CHECKING, Any, cast +from uuid import uuid4, uuid7 + +import boto3 +import pytest +from fuzzforge_common.sandboxes.engines.podman.configuration import PodmanConfiguration +from fuzzforge_common.storage.configuration import StorageConfiguration +from fuzzforge_sdk.constants import ( + FUZZFORGE_MODULE_DESCRIPTION_LENGTH_MAX, + FUZZFORGE_MODULE_NAME_LENGTH_MAX, + FUZZFORGE_MODULE_NAME_LENGTH_MIN, + FUZZFORGE_PROJECT_DESCRIPTION_LENGTH_MAX, + FUZZFORGE_PROJECT_NAME_LENGTH_MAX, + FUZZFORGE_PROJECT_NAME_LENGTH_MIN, + FUZZFORGE_WORKFLOW_DESCRIPTION_LENGTH_MAX, + FUZZFORGE_WORKFLOW_NAME_LENGTH_MAX, + FUZZFORGE_WORKFLOW_NAME_LENGTH_MIN, +) +from podman import PodmanClient +from testcontainers.minio import MinioContainer + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + from pathlib import Path + + from fuzzforge_types import ( + FuzzForgeExecutionIdentifier, + FuzzForgeModuleIdentifier, + FuzzForgeProjectIdentifier, + FuzzForgeWorkflowIdentifier, + ) + + +MINIO_DEFAULT_IMAGE: str = "minio/minio:RELEASE.2025-09-07T16-13-09Z" + + +def generate_random_string( + min_length: int, + max_length: int, +) -> str: + """TODO.""" + return "".join(random.choices(population=string.printable, k=random.randint(min_length, max_length))) # noqa: S311 + + +# ===== Project Fixtures ===== +# Note: random_project_identifier is provided by fuzzforge-tests +# Note: random_module_execution_identifier is provided by fuzzforge-tests + + +@pytest.fixture +def random_project_name() -> Callable[[], str]: + """Generate random project names.""" + + def inner() -> str: + return generate_random_string( + min_length=FUZZFORGE_PROJECT_NAME_LENGTH_MIN, + max_length=FUZZFORGE_PROJECT_NAME_LENGTH_MAX, + ) + + return inner + + +@pytest.fixture +def random_project_description() -> Callable[[], str]: + """Generate random project descriptions.""" + + def inner() -> str: + return generate_random_string( + min_length=1, + max_length=FUZZFORGE_PROJECT_DESCRIPTION_LENGTH_MAX, + ) + + return inner + + +@pytest.fixture +def random_module_name() -> Callable[[], str]: + """Generate random module names.""" + + def inner() -> str: + return generate_random_string( + min_length=FUZZFORGE_MODULE_NAME_LENGTH_MIN, + max_length=FUZZFORGE_MODULE_NAME_LENGTH_MAX, + ) + + return inner + + +@pytest.fixture +def random_module_description() -> Callable[[], str]: + """Generate random module descriptions.""" + + def inner() -> str: + return generate_random_string( + min_length=1, + max_length=FUZZFORGE_MODULE_DESCRIPTION_LENGTH_MAX, + ) + + return inner + + +@pytest.fixture +def random_workflow_identifier() -> Callable[[], FuzzForgeWorkflowIdentifier]: + """Generate random workflow identifiers.""" + + def inner() -> FuzzForgeWorkflowIdentifier: + return uuid7() + + return inner + + +@pytest.fixture +def random_workflow_name() -> Callable[[], str]: + """Generate random workflow names.""" + + def inner() -> str: + return generate_random_string( + min_length=FUZZFORGE_WORKFLOW_NAME_LENGTH_MIN, + max_length=FUZZFORGE_WORKFLOW_NAME_LENGTH_MAX, + ) + + return inner + + +@pytest.fixture +def random_workflow_description() -> Callable[[], str]: + """Generate random workflow descriptions.""" + + def inner() -> str: + return generate_random_string( + min_length=1, + max_length=FUZZFORGE_WORKFLOW_DESCRIPTION_LENGTH_MAX, + ) + + return inner + + +@pytest.fixture +def random_workflow_execution_identifier() -> Callable[[], FuzzForgeExecutionIdentifier]: + """Generate random workflow execution identifiers.""" + + def inner() -> FuzzForgeExecutionIdentifier: + return uuid7() + + return inner + + +@pytest.fixture +def random_project_identifier() -> Callable[[], FuzzForgeProjectIdentifier]: + """Generate random project identifiers. + + Returns a callable that generates fresh UUID7 identifiers for each call. + This pattern allows generating multiple unique identifiers within a single test. + + :return: Callable that generates project identifiers. + + """ + + def inner() -> FuzzForgeProjectIdentifier: + return uuid7() + + return inner + + +@pytest.fixture +def random_module_identifier() -> Callable[[], FuzzForgeModuleIdentifier]: + """Generate random module identifiers.""" + + def inner() -> FuzzForgeModuleIdentifier: + return uuid7() + + return inner + + +@pytest.fixture +def random_module_execution_identifier() -> Callable[[], FuzzForgeExecutionIdentifier]: + """Generate random workflow execution identifiers. + + Returns a callable that generates fresh UUID7 identifiers for each call. + This pattern allows generating multiple unique identifiers within a single test. + + :return: Callable that generates execution identifiers. + + """ + + def inner() -> FuzzForgeExecutionIdentifier: + return uuid7() + + return inner + + +@pytest.fixture(scope="session") +def minio_container() -> Generator[MinioContainer]: + """Provide MinIO testcontainer for test session. + + Creates a MinIO container that persists for the entire test session. + All tests share the same container but use different buckets/keys. + + :return: MinIO container instance. + + """ + with MinioContainer(image=MINIO_DEFAULT_IMAGE) as container: + yield container + + +@pytest.fixture +def minio_container_configuration(minio_container: MinioContainer) -> dict[str, str]: + """TODO.""" + return cast("dict[str, str]", minio_container.get_config()) + + +@pytest.fixture +def storage_configuration(minio_container_configuration: dict[str, str]) -> StorageConfiguration: + """Provide S3 storage backend connected to MinIO testcontainer. + + Creates the bucket in MinIO before returning the backend instance. + + :param minio_container: MinIO testcontainer fixture. + :return: Configured S3StorageBackend instance with bucket already created. + + """ + return StorageConfiguration( + endpoint=f"http://{minio_container_configuration['endpoint']}", + access_key=minio_container_configuration["access_key"], + secret_key=minio_container_configuration["secret_key"], + ) + + +@pytest.fixture +def boto3_client(minio_container_configuration: dict[str, str]) -> Any: + """TODO.""" + return boto3.client( + "s3", + endpoint_url=f"http://{minio_container_configuration['endpoint']}", + aws_access_key_id=minio_container_configuration["access_key"], + aws_secret_access_key=minio_container_configuration["secret_key"], + ) + + +@pytest.fixture +def random_bucket( + boto3_client: Any, + random_project_identifier: Callable[[], FuzzForgeProjectIdentifier], +) -> str: + """TODO.""" + project_identifier: FuzzForgeProjectIdentifier = random_project_identifier() + boto3_client.create_bucket(Bucket=str(project_identifier)) + return str(project_identifier) + + +@pytest.fixture +def podman_socket() -> str: + """TODO.""" + socket: str = environ.get("DOCKER_HOST", "") + return socket + + +@pytest.fixture +def podman_client(podman_socket: str) -> Generator[PodmanClient]: + """TODO.""" + with PodmanClient(base_url=podman_socket) as client: + yield client + + +@pytest.fixture +def podman_engine_configuration(podman_socket: str) -> PodmanConfiguration: + """TODO.""" + return PodmanConfiguration(socket=podman_socket) + + +DOCKERFILE: str = 'FROM docker.io/debian:trixie\nCMD ["/bin/sh"]' + + +@pytest.fixture +def path_to_oci(podman_client: PodmanClient, tmp_path: Path) -> Generator[Path]: + """TODO.""" + dockerfile: Path = tmp_path / "Dockerfile" + dockerfile.write_text(DOCKERFILE) + identifier = str(uuid4()) + image, _ = podman_client.images.build( + path=tmp_path, + dockerfile=dockerfile.name, + tag=identifier, + ) + path: Path = tmp_path / "image.oci" + with path.open(mode="wb") as file: + for chunk in image.save(): + file.write(chunk) + podman_client.images.get(name=identifier).remove() + yield path + path.unlink(missing_ok=True) diff --git a/fuzzforge-types/Makefile b/fuzzforge-types/Makefile new file mode 100644 index 0000000..df80d02 --- /dev/null +++ b/fuzzforge-types/Makefile @@ -0,0 +1,33 @@ +PACKAGE=$(word 1, $(shell uv version)) +VERSION=$(word 2, $(shell uv version)) + +ARTIFACTS?=./dist +SOURCES=./src + +.PHONY: clean format mypy ruff version wheel + +clean: + @find . -type d \( \ + -name '*.egg-info' \ + -o -name '.mypy_cache' \ + -o -name '.ruff_cache' \ + -o -name '__pycache__' \ + \) -printf 'removing directory %p\n' -exec rm -rf {} + + +cloc: + cloc $(SOURCES) + +format: + uv run ruff format $(SOURCES) + +mypy: + uv run mypy $(SOURCES) + +ruff: + uv run ruff check --fix $(SOURCES) + +version: + @echo '$(PACKAGE)@$(VERSION)' + +wheel: + uv build --out-dir $(ARTIFACTS) diff --git a/fuzzforge-types/README.md b/fuzzforge-types/README.md new file mode 100644 index 0000000..0b5d5d1 --- /dev/null +++ b/fuzzforge-types/README.md @@ -0,0 +1,3 @@ +# FuzzForge types + +... diff --git a/fuzzforge-types/mypy.ini b/fuzzforge-types/mypy.ini new file mode 100644 index 0000000..84e90d2 --- /dev/null +++ b/fuzzforge-types/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +plugins = pydantic.mypy +strict = True +warn_unused_ignores = True +warn_redundant_casts = True +warn_return_any = True diff --git a/fuzzforge-types/pyproject.toml b/fuzzforge-types/pyproject.toml new file mode 100644 index 0000000..2d4f634 --- /dev/null +++ b/fuzzforge-types/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "fuzzforge-types" +version = "0.0.1" +description = "Collection of types for the FuzzForge API." +authors = [] +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "pydantic==2.12.4", +] + +[project.optional-dependencies] +lints = [ + "bandit==1.8.6", + "mypy==1.18.2", + "ruff==0.14.4", +] diff --git a/fuzzforge-types/ruff.toml b/fuzzforge-types/ruff.toml new file mode 100644 index 0000000..6374f62 --- /dev/null +++ b/fuzzforge-types/ruff.toml @@ -0,0 +1,19 @@ +line-length = 120 + +[lint] +select = [ "ALL" ] +ignore = [ + "COM812", # conflicts with the formatter + "D100", # ignoring missing docstrings in public modules + "D104", # ignoring missing docstrings in public packages + "D203", # conflicts with 'D211' + "D213", # conflicts with 'D212' + "TD002", # ignoring missing author in 'TODO' statements + "TD003", # ignoring missing issue link in 'TODO' statements +] + +[lint.per-file-ignores] +"tests/*" = [ + "PLR2004", # allowing comparisons using unamed numerical constants in tests + "S101", # allowing 'assert' statements in tests +] diff --git a/fuzzforge-types/src/fuzzforge_types/__init__.py b/fuzzforge-types/src/fuzzforge_types/__init__.py new file mode 100644 index 0000000..1e240a8 --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/__init__.py @@ -0,0 +1,37 @@ +"""FuzzForge types package. + +This package exports all public types used across FuzzForge components. + +""" + +from fuzzforge_types.definitions import ( + FuzzForgeDefinitionIdentifier, + FuzzForgeDefinitionTypes, +) +from fuzzforge_types.executions import ( + FuzzForgeExecution, + FuzzForgeExecutionError, + FuzzForgeExecutionIdentifier, + FuzzForgeExecutionIncludeFilter, + FuzzForgeExecutionStatus, +) +from fuzzforge_types.identifiers import FuzzForgeProjectIdentifier +from fuzzforge_types.modules import FuzzForgeModule, FuzzForgeModuleIdentifier +from fuzzforge_types.projects import FuzzForgeProject +from fuzzforge_types.workflows import FuzzForgeWorkflow, FuzzForgeWorkflowIdentifier + +__all__ = [ + "FuzzForgeDefinitionIdentifier", + "FuzzForgeDefinitionTypes", + "FuzzForgeExecution", + "FuzzForgeExecutionError", + "FuzzForgeExecutionIdentifier", + "FuzzForgeExecutionIncludeFilter", + "FuzzForgeExecutionStatus", + "FuzzForgeModule", + "FuzzForgeModuleIdentifier", + "FuzzForgeProject", + "FuzzForgeProjectIdentifier", + "FuzzForgeWorkflow", + "FuzzForgeWorkflowIdentifier", +] diff --git a/fuzzforge-types/src/fuzzforge_types/bases.py b/fuzzforge-types/src/fuzzforge_types/bases.py new file mode 100644 index 0000000..ea54b76 --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/bases.py @@ -0,0 +1,11 @@ +"""TODO.""" + +from pydantic import BaseModel + + +class Base(BaseModel): + """TODO.""" + + model_config = { + "from_attributes": True, + } diff --git a/fuzzforge-types/src/fuzzforge_types/definitions.py b/fuzzforge-types/src/fuzzforge_types/definitions.py new file mode 100644 index 0000000..730b5b3 --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/definitions.py @@ -0,0 +1,26 @@ +"""Definition types for FuzzForge. + +This module defines the base types and enums for FuzzForge definitions, +including modules and workflows. + +""" + +from enum import StrEnum + +from pydantic import UUID7 + + +class FuzzForgeDefinitionTypes(StrEnum): + """Kind of FuzzForge definition. + + Discriminator enum used to distinguish between module and workflow definitions + in the unified definitions table. + + """ + + MODULE_DEFINITION = "module" + WORKFLOW_DEFINITION = "workflow" + + +# Type aliases for definition identifiers +type FuzzForgeDefinitionIdentifier = UUID7 diff --git a/fuzzforge-types/src/fuzzforge_types/executions.py b/fuzzforge-types/src/fuzzforge_types/executions.py new file mode 100644 index 0000000..2aa7373 --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/executions.py @@ -0,0 +1,75 @@ +"""TODO.""" + +from datetime import datetime # noqa: TC003 +from enum import StrEnum + +from pydantic import UUID7, Field + +from fuzzforge_types.bases import Base +from fuzzforge_types.definitions import FuzzForgeDefinitionIdentifier, FuzzForgeDefinitionTypes # noqa: TC001 +from fuzzforge_types.identifiers import FuzzForgeProjectIdentifier # noqa: TC001 + + +class FuzzForgeExecutionStatus(StrEnum): + """TODO.""" + + PENDING = "PENDING" + RUNNING = "RUNNING" + FINISHED = "FINISHED" + + +class FuzzForgeExecutionError(StrEnum): + """TODO.""" + + GENERIC_ERROR = "GENERIC_ERROR" + + +class FuzzForgeExecutionIncludeFilter(StrEnum): + """Filter for including specific execution types when listing. + + Used to filter executions by their definition kind (module or workflow). + This filter is required when listing executions to ensure explicit intent. + + """ + + ALL = "all" + MODULES = "modules" + WORKFLOWS = "workflows" + + +# Type alias for unified execution identifiers +type FuzzForgeExecutionIdentifier = UUID7 + + +class FuzzForgeExecution(Base): + """DTO for unified execution data. + + Represents both module and workflow executions in a single model. + The definition_kind field discriminates between the two types. + + """ + + execution_identifier: FuzzForgeExecutionIdentifier = Field( + description="The identifier of this execution.", + ) + execution_status: FuzzForgeExecutionStatus = Field( + description="The current status of the execution.", + ) + execution_error: FuzzForgeExecutionError | None = Field( + description="The error associated with the execution, if any.", + ) + project_identifier: FuzzForgeProjectIdentifier = Field( + description="The identifier of the project this execution belongs to.", + ) + definition_identifier: FuzzForgeDefinitionIdentifier = Field( + description="The identifier of the definition (module or workflow) being executed.", + ) + definition_kind: FuzzForgeDefinitionTypes = Field( + description="The kind of definition being executed (module or workflow).", + ) + created_at: datetime = Field( + description="The creation date of the execution.", + ) + updated_at: datetime = Field( + description="The latest modification date of the execution.", + ) diff --git a/fuzzforge-types/src/fuzzforge_types/identifiers.py b/fuzzforge-types/src/fuzzforge_types/identifiers.py new file mode 100644 index 0000000..a9af833 --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/identifiers.py @@ -0,0 +1,5 @@ +"""TODO.""" + +from pydantic import UUID7 + +type FuzzForgeProjectIdentifier = UUID7 diff --git a/fuzzforge-types/src/fuzzforge_types/modules.py b/fuzzforge-types/src/fuzzforge_types/modules.py new file mode 100644 index 0000000..373836e --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/modules.py @@ -0,0 +1,30 @@ +"""TODO.""" + +from datetime import datetime # noqa: TC003 + +from pydantic import Field + +from fuzzforge_types.bases import Base +from fuzzforge_types.definitions import FuzzForgeDefinitionIdentifier + +type FuzzForgeModuleIdentifier = FuzzForgeDefinitionIdentifier + + +class FuzzForgeModule(Base): + """TODO.""" + + module_description: str = Field( + description="The description of the module.", + ) + module_identifier: FuzzForgeModuleIdentifier = Field( + description="The identifier of the module.", + ) + module_name: str = Field( + description="The name of the module.", + ) + created_at: datetime = Field( + description="The creation date of the module.", + ) + updated_at: datetime = Field( + description="The latest modification date of the module.", + ) diff --git a/fuzzforge-types/src/fuzzforge_types/projects.py b/fuzzforge-types/src/fuzzforge_types/projects.py new file mode 100644 index 0000000..2821861 --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/projects.py @@ -0,0 +1,34 @@ +"""TODO.""" + +from datetime import datetime # noqa: TC003 + +from pydantic import Field + +from fuzzforge_types.bases import Base +from fuzzforge_types.executions import FuzzForgeExecution # noqa: TC001 +from fuzzforge_types.identifiers import FuzzForgeProjectIdentifier # noqa: TC001 + + +class FuzzForgeProject(Base): + """TODO.""" + + project_description: str = Field( + description="The description of the project.", + ) + project_identifier: FuzzForgeProjectIdentifier = Field( + description="The identifier of the project.", + ) + project_name: str = Field( + description="The name of the project.", + ) + created_at: datetime = Field( + description="The creation date of the project.", + ) + updated_at: datetime = Field( + description="The latest modification date of the project.", + ) + + executions: list[FuzzForgeExecution] | None = Field( + default=None, + description="The module and workflow executions associated with the project.", + ) diff --git a/fuzzforge-types/src/fuzzforge_types/py.typed b/fuzzforge-types/src/fuzzforge_types/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/fuzzforge-types/src/fuzzforge_types/workflows.py b/fuzzforge-types/src/fuzzforge_types/workflows.py new file mode 100644 index 0000000..172db5c --- /dev/null +++ b/fuzzforge-types/src/fuzzforge_types/workflows.py @@ -0,0 +1,30 @@ +"""TODO.""" + +from datetime import datetime # noqa: TC003 + +from pydantic import Field + +from fuzzforge_types.bases import Base +from fuzzforge_types.definitions import FuzzForgeDefinitionIdentifier + +type FuzzForgeWorkflowIdentifier = FuzzForgeDefinitionIdentifier + + +class FuzzForgeWorkflow(Base): + """TODO.""" + + workflow_description: str = Field( + description="The description of the workflow.", + ) + workflow_identifier: FuzzForgeWorkflowIdentifier = Field( + description="The identifier of the workflow.", + ) + workflow_name: str = Field( + description="The name of the workflow.", + ) + created_at: datetime = Field( + description="The creation date of the workflow.", + ) + updated_at: datetime = Field( + description="The latest modification date of the workflow.", + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bac9374 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "fuzzforge-oss" +version = "1.0.0" +description = "FuzzForge OSS - AI-driven security research platform for local execution" +readme = "README.md" +requires-python = ">=3.14" +authors = [ + { name = "FuzzingLabs", email = "contact@fuzzinglabs.com" } +] + +[tool.uv.workspace] +members = [ + "fuzzforge-common", + "fuzzforge-types", + "fuzzforge-modules/fuzzforge-modules-sdk", + "fuzzforge-runner", + "fuzzforge-mcp", + "fuzzforge-cli", + "fuzzforge-tests", +] + +[tool.uv.sources] +fuzzforge-common = { workspace = true } +fuzzforge-types = { workspace = true } +fuzzforge-modules-sdk = { workspace = true } +fuzzforge-runner = { workspace = true } +fuzzforge-mcp = { workspace = true } +fuzzforge-cli = { workspace = true } +fuzzforge-tests = { workspace = true }