feat: FuzzForge AI - complete rewrite for OSS release

This commit is contained in:
AFredefon
2026-01-30 09:57:48 +01:00
commit b46f050aef
226 changed files with 12943 additions and 0 deletions

42
fuzzforge-common/Makefile Normal file
View File

@@ -0,0 +1,42 @@
PACKAGE=$(word 1, $(shell uv version))
VERSION=$(word 2, $(shell uv version))
ARTIFACTS?=./dist
SOURCES=./src
TESTS=./tests
.PHONY: bandit clean format mypy pytest ruff version wheel
bandit:
uv run bandit --recursive $(SOURCES)
clean:
@find . -type d \( \
-name '*.egg-info' \
-o -name '.mypy_cache' \
-o -name '.pytest_cache' \
-o -name '.ruff_cache' \
-o -name '__pycache__' \
\) -printf 'removing directory %p\n' -exec rm -rf {} +
cloc:
cloc $(SOURCES)
format:
uv run ruff format $(SOURCES) $(TESTS)
mypy:
uv run mypy $(SOURCES)
pytest:
uv run pytest $(TESTS)
ruff:
uv run ruff check --fix $(SOURCES) $(TESTS)
version:
@echo '$(PACKAGE)@$(VERSION)'
wheel:
uv build --out-dir $(ARTIFACTS)

View File

@@ -0,0 +1,3 @@
# FuzzForge Common
...

12
fuzzforge-common/mypy.ini Normal file
View File

@@ -0,0 +1,12 @@
[mypy]
plugins = pydantic.mypy
strict = True
warn_unused_ignores = True
warn_redundant_casts = True
warn_return_any = True
[mypy-botocore.*]
ignore_missing_imports = True
[mypy-boto3.*]
ignore_missing_imports = True

View File

@@ -0,0 +1,25 @@
[project]
name = "fuzzforge-common"
version = "0.0.1"
description = "FuzzForge's common types and utilities."
authors = []
readme = "README.md"
requires-python = ">=3.14"
dependencies = [
"fuzzforge-types==0.0.1",
"podman==5.6.0",
"pydantic==2.12.4",
]
[project.optional-dependencies]
lints = [
"bandit==1.8.6",
"mypy==1.18.2",
"ruff==0.14.4",
]
tests = [
"pytest==9.0.2",
]
[tool.uv.sources]
fuzzforge-types = { workspace = true }

View File

@@ -0,0 +1,3 @@
[pytest]
env =
DOCKER_HOST=unix:///run/user/1000/podman/podman.sock

View File

@@ -0,0 +1,20 @@
line-length = 120
[lint]
select = [ "ALL" ]
ignore = [
"COM812", # conflicts with the formatter
"D100", # ignoring missing docstrings in public modules
"D104", # ignoring missing docstrings in public packages
"D203", # conflicts with 'D211'
"D213", # conflicts with 'D212'
"TD002", # ignoring missing author in 'TODO' statements
"TD003", # ignoring missing issue link in 'TODO' statements
]
[lint.per-file-ignores]
"tests/*" = [
"ANN401", # allowing 'typing.Any' to be used to type function parameters in tests
"PLR2004", # allowing comparisons using unamed numerical constants in tests
"S101", # allowing 'assert' statements in tests
]

View File

@@ -0,0 +1,54 @@
"""FuzzForge Common - Shared abstractions and implementations for FuzzForge.
This package provides:
- Sandbox engine abstractions (Podman, Docker)
- Storage abstractions (S3) - requires 'storage' extra
- Common exceptions
Example usage:
from fuzzforge_common import (
AbstractFuzzForgeSandboxEngine,
ImageInfo,
Podman,
PodmanConfiguration,
)
# For storage (requires boto3):
from fuzzforge_common.storage import Storage
"""
from fuzzforge_common.exceptions import FuzzForgeError
from fuzzforge_common.sandboxes import (
AbstractFuzzForgeEngineConfiguration,
AbstractFuzzForgeSandboxEngine,
Docker,
DockerConfiguration,
FuzzForgeSandboxEngines,
ImageInfo,
Podman,
PodmanConfiguration,
)
# Storage exceptions are always available (no boto3 required)
from fuzzforge_common.storage.exceptions import (
FuzzForgeStorageError,
StorageConnectionError,
StorageDownloadError,
StorageUploadError,
)
__all__ = [
"AbstractFuzzForgeEngineConfiguration",
"AbstractFuzzForgeSandboxEngine",
"Docker",
"DockerConfiguration",
"FuzzForgeError",
"FuzzForgeSandboxEngines",
"FuzzForgeStorageError",
"ImageInfo",
"Podman",
"PodmanConfiguration",
"StorageConnectionError",
"StorageDownloadError",
"StorageUploadError",
]

View File

@@ -0,0 +1,24 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import Any
class FuzzForgeError(Exception):
"""Base exception for all FuzzForge custom exceptions.
All domain exceptions should inherit from this base to enable
consistent exception handling and hierarchy navigation.
"""
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
"""Initialize FuzzForge error.
:param message: Error message.
:param details: Optional error details dictionary.
"""
Exception.__init__(self, message)
self.message = message
self.details = details or {}

View File

@@ -0,0 +1,23 @@
"""FuzzForge sandbox abstractions and implementations."""
from fuzzforge_common.sandboxes.engines import (
AbstractFuzzForgeEngineConfiguration,
AbstractFuzzForgeSandboxEngine,
Docker,
DockerConfiguration,
FuzzForgeSandboxEngines,
ImageInfo,
Podman,
PodmanConfiguration,
)
__all__ = [
"AbstractFuzzForgeEngineConfiguration",
"AbstractFuzzForgeSandboxEngine",
"Docker",
"DockerConfiguration",
"FuzzForgeSandboxEngines",
"ImageInfo",
"Podman",
"PodmanConfiguration",
]

View File

@@ -0,0 +1,21 @@
"""Container engine implementations for FuzzForge sandboxes."""
from fuzzforge_common.sandboxes.engines.base import (
AbstractFuzzForgeEngineConfiguration,
AbstractFuzzForgeSandboxEngine,
ImageInfo,
)
from fuzzforge_common.sandboxes.engines.docker import Docker, DockerConfiguration
from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines
from fuzzforge_common.sandboxes.engines.podman import Podman, PodmanConfiguration
__all__ = [
"AbstractFuzzForgeEngineConfiguration",
"AbstractFuzzForgeSandboxEngine",
"Docker",
"DockerConfiguration",
"FuzzForgeSandboxEngines",
"ImageInfo",
"Podman",
"PodmanConfiguration",
]

View File

@@ -0,0 +1,15 @@
"""Base engine abstractions."""
from fuzzforge_common.sandboxes.engines.base.configuration import (
AbstractFuzzForgeEngineConfiguration,
)
from fuzzforge_common.sandboxes.engines.base.engine import (
AbstractFuzzForgeSandboxEngine,
ImageInfo,
)
__all__ = [
"AbstractFuzzForgeEngineConfiguration",
"AbstractFuzzForgeSandboxEngine",
"ImageInfo",
]

View File

@@ -0,0 +1,24 @@
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from pydantic import BaseModel
from fuzzforge_common.sandboxes.engines.enumeration import (
FuzzForgeSandboxEngines, # noqa: TC001 (required by 'pydantic' at runtime)
)
if TYPE_CHECKING:
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine
class AbstractFuzzForgeEngineConfiguration(ABC, BaseModel):
"""TODO."""
#: TODO.
kind: FuzzForgeSandboxEngines
@abstractmethod
def into_engine(self) -> AbstractFuzzForgeSandboxEngine:
"""TODO."""
message: str = f"method 'into_engine' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)

View File

@@ -0,0 +1,281 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from pathlib import Path, PurePath
@dataclass
class ImageInfo:
"""Information about a container image."""
#: Full image reference (e.g., "localhost/fuzzforge-module-echidna:latest").
reference: str
#: Repository name (e.g., "localhost/fuzzforge-module-echidna").
repository: str
#: Image tag (e.g., "latest").
tag: str
#: Image ID (short hash).
image_id: str | None = None
#: Image size in bytes.
size: int | None = None
class AbstractFuzzForgeSandboxEngine(ABC):
"""Abstract class used as a base for all FuzzForge sandbox engine classes."""
@abstractmethod
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
:returns: List of ImageInfo objects for available images.
"""
message: str = f"method 'list_images' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def register_archive(self, archive: Path, repository: str) -> None:
"""TODO.
:param archive: TODO.
"""
message: str = f"method 'register_archive' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox based on the given image.
:param image: The image the sandbox should be based on.
:returns: The sandbox identifier.
"""
message: str = f"method 'spawn_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:param destination: TODO.
"""
message: str = f"method 'push_archive_to_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def start_sandbox(self, identifier: str) -> None:
"""TODO.
:param identifier: The identifier of the sandbox to start.
"""
message: str = f"method 'start_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
:param sandbox: The identifier of the sandbox.
:param command: The command to run.
"""
message: str = f"method 'execute_inside_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:returns: TODO.
"""
message: str = f"method 'pull_archive_from_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to terminate.
"""
message: str = f"method 'terminate_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
# -------------------------------------------------------------------------
# Extended Container Operations
# -------------------------------------------------------------------------
@abstractmethod
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference (e.g., "localhost/module:latest").
:returns: True if image exists, False otherwise.
"""
message: str = f"method 'image_exists' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference to pull.
:param timeout: Timeout in seconds for the pull operation.
:raises FuzzForgeError: If pull fails.
"""
message: str = f"method 'pull_image' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
message: str = f"method 'tag_image' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
message: str = f"method 'create_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for it to complete.
:param identifier: Container identifier.
:param timeout: Timeout in seconds for execution.
:returns: Tuple of (exit_code, stdout, stderr).
"""
message: str = f"method 'start_container_attached' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
message: str = f"method 'copy_to_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
message: str = f"method 'copy_from_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal even if running.
"""
message: str = f"method 'remove_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
# -------------------------------------------------------------------------
# Continuous/Background Execution Operations
# -------------------------------------------------------------------------
@abstractmethod
def start_container(self, identifier: str) -> None:
"""Start a container without waiting for it to complete (detached mode).
:param identifier: Container identifier.
"""
message: str = f"method 'start_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status (e.g., "running", "exited", "created").
"""
message: str = f"method 'get_container_status' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container gracefully.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
message: str = f"method 'stop_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a running container using exec.
:param identifier: Container identifier.
:param path: Path to file inside container.
:returns: File contents as string.
"""
message: str = f"method 'read_file_from_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
message: str = f"method 'list_containers' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)

View File

@@ -0,0 +1,11 @@
"""Docker container engine implementation."""
from fuzzforge_common.sandboxes.engines.docker.configuration import (
DockerConfiguration,
)
from fuzzforge_common.sandboxes.engines.docker.engine import Docker
__all__ = [
"Docker",
"DockerConfiguration",
]

View File

@@ -0,0 +1,22 @@
from typing import TYPE_CHECKING, Literal
from fuzzforge_common.sandboxes.engines.base.configuration import AbstractFuzzForgeEngineConfiguration
from fuzzforge_common.sandboxes.engines.docker.engine import Docker
from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines
if TYPE_CHECKING:
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine
class DockerConfiguration(AbstractFuzzForgeEngineConfiguration):
"""TODO."""
#: TODO.
kind: Literal[FuzzForgeSandboxEngines.DOCKER] = FuzzForgeSandboxEngines.DOCKER
#: TODO.
socket: str
def into_engine(self) -> AbstractFuzzForgeSandboxEngine:
"""TODO."""
return Docker(socket=self.socket)

View File

@@ -0,0 +1,174 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from pathlib import Path, PurePath
class Docker(AbstractFuzzForgeSandboxEngine):
"""TODO."""
#: TODO.
__socket: str
def __init__(self, socket: str) -> None:
"""Initialize an instance of the class.
:param socket: TODO.
"""
super().__init__()
self.__socket = socket
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
:returns: List of ImageInfo objects for available images.
"""
# TODO: Implement Docker image listing
message: str = "Docker engine list_images is not yet implemented"
raise NotImplementedError(message)
def register_archive(self, archive: Path, repository: str) -> None:
"""TODO.
:param archive: TODO.
"""
return super().register_archive(archive=archive, repository=repository)
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox based on the given image.
:param image: The image the sandbox should be based on.
:returns: The sandbox identifier.
"""
return super().spawn_sandbox(image)
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:param destination: TODO.
"""
super().push_archive_to_sandbox(identifier, source, destination)
def start_sandbox(self, identifier: str) -> None:
"""TODO.
:param identifier: The identifier of the sandbox to start.
"""
super().start_sandbox(identifier)
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
:param sandbox: The identifier of the sandbox.
:param command: The command to run.
"""
super().execute_inside_sandbox(identifier, command)
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:returns: TODO.
"""
return super().pull_archive_from_sandbox(identifier, source)
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to terminate.
"""
super().terminate_sandbox(identifier)
# -------------------------------------------------------------------------
# Extended Container Operations (stubs - not yet implemented)
# -------------------------------------------------------------------------
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally."""
message: str = "Docker engine image_exists is not yet implemented"
raise NotImplementedError(message)
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry."""
message: str = "Docker engine pull_image is not yet implemented"
raise NotImplementedError(message)
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name."""
message: str = "Docker engine tag_image is not yet implemented"
raise NotImplementedError(message)
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image."""
message: str = "Docker engine create_container is not yet implemented"
raise NotImplementedError(message)
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for it to complete."""
message: str = "Docker engine start_container_attached is not yet implemented"
raise NotImplementedError(message)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container."""
message: str = "Docker engine copy_to_container is not yet implemented"
raise NotImplementedError(message)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container."""
message: str = "Docker engine copy_from_container is not yet implemented"
raise NotImplementedError(message)
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container."""
message: str = "Docker engine remove_container is not yet implemented"
raise NotImplementedError(message)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting for it to complete."""
message: str = "Docker engine start_container is not yet implemented"
raise NotImplementedError(message)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container."""
message: str = "Docker engine get_container_status is not yet implemented"
raise NotImplementedError(message)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container gracefully."""
message: str = "Docker engine stop_container is not yet implemented"
raise NotImplementedError(message)
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a running container using exec."""
message: str = "Docker engine read_file_from_container is not yet implemented"
raise NotImplementedError(message)
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers."""
message: str = "Docker engine list_containers is not yet implemented"
raise NotImplementedError(message)

View File

@@ -0,0 +1,11 @@
from enum import StrEnum
class FuzzForgeSandboxEngines(StrEnum):
"""TODO."""
#: TODO.
DOCKER = "docker"
#: TODO.
PODMAN = "podman"

View File

@@ -0,0 +1,13 @@
"""Podman container engine implementation."""
from fuzzforge_common.sandboxes.engines.podman.cli import PodmanCLI
from fuzzforge_common.sandboxes.engines.podman.configuration import (
PodmanConfiguration,
)
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
__all__ = [
"Podman",
"PodmanCLI",
"PodmanConfiguration",
]

View File

@@ -0,0 +1,444 @@
"""Podman CLI engine with custom storage support.
This engine uses subprocess calls to the Podman CLI instead of the socket API,
allowing for custom storage paths (--root, --runroot) that work regardless of
system Podman configuration or snap environment issues.
"""
from __future__ import annotations
import json
import subprocess
import tarfile
from io import BytesIO
from pathlib import Path, PurePath
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, cast
from fuzzforge_common.exceptions import FuzzForgeError
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structured logger."""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
class PodmanCLI(AbstractFuzzForgeSandboxEngine):
"""Podman engine using CLI with custom storage paths.
This implementation uses subprocess calls to the Podman CLI with --root
and --runroot flags, providing isolation from system Podman storage.
This is particularly useful when running from VS Code snap which sets
XDG_DATA_HOME to a version-specific path.
"""
__graphroot: Path
__runroot: Path
def __init__(self, graphroot: Path, runroot: Path) -> None:
"""Initialize the PodmanCLI engine.
:param graphroot: Path to container image storage.
:param runroot: Path to container runtime state.
"""
AbstractFuzzForgeSandboxEngine.__init__(self)
self.__graphroot = graphroot
self.__runroot = runroot
# Ensure directories exist
self.__graphroot.mkdir(parents=True, exist_ok=True)
self.__runroot.mkdir(parents=True, exist_ok=True)
def _base_cmd(self) -> list[str]:
"""Get base Podman command with storage flags.
:returns: Base command list with --root and --runroot.
"""
return [
"podman",
"--root", str(self.__graphroot),
"--runroot", str(self.__runroot),
]
def _run(self, args: list[str], *, check: bool = True, capture: bool = True) -> subprocess.CompletedProcess:
"""Run a Podman command.
:param args: Command arguments (without 'podman').
:param check: Raise exception on non-zero exit.
:param capture: Capture stdout/stderr.
:returns: CompletedProcess result.
"""
cmd = self._base_cmd() + args
get_logger().debug("running podman command", cmd=" ".join(cmd))
return subprocess.run(
cmd,
check=check,
capture_output=capture,
text=True,
)
# -------------------------------------------------------------------------
# Image Operations
# -------------------------------------------------------------------------
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images.
:returns: List of ImageInfo objects.
"""
result = self._run(["images", "--format", "json"])
images: list[ImageInfo] = []
try:
data = json.loads(result.stdout) if result.stdout.strip() else []
except json.JSONDecodeError:
get_logger().warning("failed to parse podman images output")
return images
for image in data:
# Get repository and tag from Names
names = image.get("Names") or []
for name in names:
if filter_prefix and not name.startswith(filter_prefix):
continue
# Parse repository and tag
if ":" in name:
repo, tag = name.rsplit(":", 1)
else:
repo = name
tag = "latest"
images.append(
ImageInfo(
reference=name,
repository=repo,
tag=tag,
image_id=image.get("Id", "")[:12],
size=image.get("Size"),
)
)
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
return images
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference.
:returns: True if image exists.
"""
result = self._run(["image", "exists", image], check=False)
return result.returncode == 0
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference.
:param timeout: Timeout in seconds.
"""
get_logger().info("pulling image", image=image)
try:
self._run(["pull", image])
get_logger().info("image pulled successfully", image=image)
except subprocess.CalledProcessError as exc:
message = f"Failed to pull image '{image}': {exc.stderr}"
raise FuzzForgeError(message) from exc
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
self._run(["tag", source, target])
get_logger().debug("tagged image", source=source, target=target)
def build_image(self, context_path: Path, tag: str, dockerfile: str = "Dockerfile") -> None:
"""Build an image from a Dockerfile.
:param context_path: Path to build context.
:param tag: Image tag.
:param dockerfile: Dockerfile name.
"""
get_logger().info("building image", tag=tag, context=str(context_path))
self._run(["build", "-t", tag, "-f", dockerfile, str(context_path)])
get_logger().info("image built successfully", tag=tag)
def register_archive(self, archive: Path, repository: str) -> None:
"""Load an image from a tar archive.
:param archive: Path to tar archive.
:param repository: Repository name for the loaded image.
"""
result = self._run(["load", "-i", str(archive)])
# Tag the loaded image
# Parse loaded image ID from output
for line in result.stdout.splitlines():
if "Loaded image:" in line:
loaded_image = line.split("Loaded image:")[-1].strip()
self._run(["tag", loaded_image, f"{repository}:latest"])
break
get_logger().debug("registered archive", archive=str(archive), repository=repository)
# -------------------------------------------------------------------------
# Container Operations
# -------------------------------------------------------------------------
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox (container) from an image.
:param image: Image to create container from.
:returns: Container identifier.
"""
result = self._run(["create", image])
container_id = result.stdout.strip()
get_logger().debug("created container", container_id=container_id)
return container_id
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
args = ["create"]
if volumes:
for host_path, container_path in volumes.items():
args.extend(["-v", f"{host_path}:{container_path}:ro"])
args.append(image)
result = self._run(args)
container_id = result.stdout.strip()
get_logger().debug("created container", container_id=container_id, image=image)
return container_id
def start_sandbox(self, identifier: str) -> None:
"""Start a container.
:param identifier: Container identifier.
"""
self._run(["start", identifier])
get_logger().debug("started container", container_id=identifier)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting.
:param identifier: Container identifier.
"""
self._run(["start", identifier])
get_logger().debug("started container (detached)", container_id=identifier)
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for completion.
:param identifier: Container identifier.
:param timeout: Timeout in seconds.
:returns: Tuple of (exit_code, stdout, stderr).
"""
get_logger().debug("starting container attached", container_id=identifier)
# Start the container
self._run(["start", identifier])
# Wait for completion
wait_result = self._run(["wait", identifier])
exit_code = int(wait_result.stdout.strip()) if wait_result.stdout.strip() else -1
# Get logs
stdout_result = self._run(["logs", identifier], check=False)
stdout_str = stdout_result.stdout or ""
stderr_str = stdout_result.stderr or ""
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
return (exit_code, stdout_str, stderr_str)
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside a container.
:param identifier: Container identifier.
:param command: Command to run.
"""
get_logger().debug("executing command in container", container_id=identifier)
self._run(["exec", identifier] + command)
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""Copy an archive to a container.
:param identifier: Container identifier.
:param source: Source archive path.
:param destination: Destination path in container.
"""
get_logger().debug("copying to container", container_id=identifier, source=str(source))
self._run(["cp", str(source), f"{identifier}:{destination}"])
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""Copy files from a container to a local archive.
:param identifier: Container identifier.
:param source: Source path in container.
:returns: Path to local archive.
"""
get_logger().debug("copying from container", container_id=identifier, source=str(source))
with NamedTemporaryFile(delete=False, delete_on_close=False, suffix=".tar") as tmp:
self._run(["cp", f"{identifier}:{source}", tmp.name])
return Path(tmp.name)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
self._run(["cp", str(source), f"{identifier}:{destination}"])
get_logger().debug("copied to container", source=str(source), destination=destination)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
destination.mkdir(parents=True, exist_ok=True)
self._run(["cp", f"{identifier}:{source}", str(destination)])
get_logger().debug("copied from container", source=source, destination=str(destination))
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate and remove a container.
:param identifier: Container identifier.
"""
# Stop if running
self._run(["stop", identifier], check=False)
# Remove
self._run(["rm", "-f", identifier], check=False)
get_logger().debug("terminated container", container_id=identifier)
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal.
"""
args = ["rm"]
if force:
args.append("-f")
args.append(identifier)
self._run(args, check=False)
get_logger().debug("removed container", container_id=identifier)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
self._run(["stop", "-t", str(timeout), identifier], check=False)
get_logger().debug("stopped container", container_id=identifier)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status.
"""
result = self._run(["inspect", "--format", "{{.State.Status}}", identifier], check=False)
return result.stdout.strip() if result.returncode == 0 else "unknown"
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a container.
:param identifier: Container identifier.
:param path: Path to file in container.
:returns: File contents.
"""
result = self._run(["exec", identifier, "cat", path], check=False)
if result.returncode != 0:
get_logger().debug("failed to read file from container", path=path)
return ""
return result.stdout
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
args = ["ps", "--format", "json"]
if all_containers:
args.append("-a")
result = self._run(args)
try:
data = json.loads(result.stdout) if result.stdout.strip() else []
# Handle both list and single object responses
if isinstance(data, dict):
data = [data]
return [
{
"Id": c.get("Id", ""),
"Names": c.get("Names", []),
"Status": c.get("State", ""),
"Image": c.get("Image", ""),
}
for c in data
]
except json.JSONDecodeError:
return []
# -------------------------------------------------------------------------
# Utility Methods
# -------------------------------------------------------------------------
def get_storage_info(self) -> dict:
"""Get storage configuration info.
:returns: Dict with graphroot and runroot paths.
"""
return {
"graphroot": str(self.__graphroot),
"runroot": str(self.__runroot),
}

View File

@@ -0,0 +1,22 @@
from typing import TYPE_CHECKING, Literal
from fuzzforge_common.sandboxes.engines.base.configuration import AbstractFuzzForgeEngineConfiguration
from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
if TYPE_CHECKING:
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine
class PodmanConfiguration(AbstractFuzzForgeEngineConfiguration):
"""TODO."""
#: TODO.
kind: Literal[FuzzForgeSandboxEngines.PODMAN] = FuzzForgeSandboxEngines.PODMAN
#: TODO.
socket: str
def into_engine(self) -> AbstractFuzzForgeSandboxEngine:
"""TODO."""
return Podman(socket=self.socket)

View File

@@ -0,0 +1,496 @@
from __future__ import annotations
import tarfile
from io import BytesIO
from pathlib import Path, PurePath
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, cast
from podman.errors import ImageNotFound
from fuzzforge_common.exceptions import FuzzForgeError
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from podman import PodmanClient
from podman.domain.containers import Container
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""TODO."""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
class Podman(AbstractFuzzForgeSandboxEngine):
"""TODO."""
#: TODO.
__socket: str
def __init__(self, socket: str) -> None:
"""Initialize an instance of the class.
:param socket: TODO.
"""
AbstractFuzzForgeSandboxEngine.__init__(self)
self.__socket = socket
def get_client(self) -> PodmanClient:
"""TODO.
:returns TODO.
"""
from podman import PodmanClient # noqa: PLC0415 (required by temporal)
return PodmanClient(base_url=self.__socket)
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
:returns: List of ImageInfo objects for available images.
"""
client: PodmanClient = self.get_client()
images: list[ImageInfo] = []
with client:
for image in client.images.list():
# Get all tags for this image
tags = image.tags or []
for tag in tags:
# Apply filter if specified
if filter_prefix and not tag.startswith(filter_prefix):
continue
# Parse repository and tag
if ":" in tag:
repo, tag_name = tag.rsplit(":", 1)
else:
repo = tag
tag_name = "latest"
images.append(
ImageInfo(
reference=tag,
repository=repo,
tag=tag_name,
image_id=image.short_id if hasattr(image, "short_id") else image.id[:12],
size=image.attrs.get("Size") if hasattr(image, "attrs") else None,
)
)
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
return images
def register_archive(self, archive: Path, repository: str) -> None:
"""TODO.
:param archive: TODO.
"""
client: PodmanClient = self.get_client()
with client:
images = list(client.images.load(file_path=archive))
if len(images) != 1:
message: str = "expected only one image"
raise FuzzForgeError(message)
image = images[0]
image.tag(repository=repository, tag="latest")
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox based on the given image.
:param image: The image the sandbox should be based on.
:returns: The sandbox identifier.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.create(image=image)
container_identifier: str = container.id
get_logger().debug("create podman container", container_identifier=container_identifier)
return container_identifier
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:param destination: TODO.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"push archive to podman container",
container_identifier=identifier,
container_status=container.status,
)
# reading everything at once for now, even though this temporary solution is not viable with large files,
# since the podman sdk does not currently expose a way to chunk uploads.
# in order to fix this issue, we could directly interact with the podman rest api or make a contribution
# to the podman sdk in order to allow the 'put_archive' method to support chunked uploads.
data: bytes = source.read_bytes()
container.put_archive(path=str(destination), data=data)
def start_sandbox(self, identifier: str) -> None:
"""Start the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to start.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"start podman container",
container_identifier=identifier,
container_status=container.status,
)
container.start()
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
:param sandbox: The identifier of the sandbox.
:param command: The command to run.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"executing command inside podman container",
container_identifier=identifier,
container_status=container.status,
)
(status, (stdout, stderr)) = container.exec_run(cmd=command, demux=True)
get_logger().debug(
"command execution result",
status=status,
stdout_size=len(stdout) if stdout else 0,
stderr_size=len(stderr) if stderr else 0,
)
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:returns: TODO.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"pull archive from podman container",
container_identifier=identifier,
container_status=container.status,
)
with NamedTemporaryFile(delete=False, delete_on_close=False) as file:
stream, _stat = container.get_archive(path=str(source))
for chunk in stream:
file.write(chunk)
get_logger().debug(
"created archive",
archive=file.name,
)
return Path(file.name)
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to terminate.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"kill podman container",
container_identifier=identifier,
container_status=container.status,
)
# Only kill running containers; for created/stopped, skip to remove
if container.status in ("running", "paused"):
container.kill()
get_logger().debug(
"remove podman container",
container_identifier=identifier,
container_status=container.status,
)
container.remove()
# -------------------------------------------------------------------------
# Extended Container Operations
# -------------------------------------------------------------------------
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference (e.g., "localhost/module:latest").
:returns: True if image exists, False otherwise.
"""
client: PodmanClient = self.get_client()
with client:
try:
client.images.get(name=image)
except ImageNotFound:
return False
else:
return True
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference to pull.
:param timeout: Timeout in seconds for the pull operation.
:raises FuzzForgeError: If pull fails.
"""
client: PodmanClient = self.get_client()
with client:
try:
get_logger().info("pulling image", image=image)
client.images.pull(repository=image)
get_logger().info("image pulled successfully", image=image)
except Exception as exc:
message = f"Failed to pull image '{image}': {exc}"
raise FuzzForgeError(message) from exc
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
client: PodmanClient = self.get_client()
with client:
image = client.images.get(name=source)
# Parse target into repository and tag
if ":" in target:
repo, tag = target.rsplit(":", 1)
else:
repo = target
tag = "latest"
image.tag(repository=repo, tag=tag)
get_logger().debug("tagged image", source=source, target=target)
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
client: PodmanClient = self.get_client()
with client:
# Build volume mounts in podman format
mounts = []
if volumes:
for host_path, container_path in volumes.items():
mounts.append({"type": "bind", "source": host_path, "target": container_path, "read_only": True})
container: Container = client.containers.create(image=image, mounts=mounts if mounts else None)
container_id: str = str(container.id)
get_logger().debug("created container", container_id=container_id, image=image)
return container_id
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for it to complete.
:param identifier: Container identifier.
:param timeout: Timeout in seconds for execution.
:returns: Tuple of (exit_code, stdout, stderr).
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug("starting container attached", container_id=identifier)
# Start the container
container.start()
# Wait for completion with timeout
result = container.wait(timeout=timeout)
exit_code: int = result.get("StatusCode", -1) if isinstance(result, dict) else int(result)
# Get logs
stdout_raw = container.logs(stdout=True, stderr=False)
stderr_raw = container.logs(stdout=False, stderr=True)
# Decode if bytes
stdout_str: str = ""
stderr_str: str = ""
if isinstance(stdout_raw, bytes):
stdout_str = stdout_raw.decode("utf-8", errors="replace")
elif isinstance(stdout_raw, str):
stdout_str = stdout_raw
if isinstance(stderr_raw, bytes):
stderr_str = stderr_raw.decode("utf-8", errors="replace")
elif isinstance(stderr_raw, str):
stderr_str = stderr_raw
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
return (exit_code, stdout_str, stderr_str)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
# Create tar archive in memory
tar_buffer = BytesIO()
with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
tar.add(str(source), arcname=Path(source).name)
tar_buffer.seek(0)
# Use put_archive to copy
container.put_archive(path=destination, data=tar_buffer.read())
get_logger().debug("copied to container", source=str(source), destination=destination)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
# Get archive from container
stream, _stat = container.get_archive(path=source)
# Write to temp file and extract
tar_buffer = BytesIO()
for chunk in stream:
tar_buffer.write(chunk)
tar_buffer.seek(0)
# Extract to destination
destination.mkdir(parents=True, exist_ok=True)
with tarfile.open(fileobj=tar_buffer, mode="r") as tar:
tar.extractall(path=destination) # noqa: S202 (trusted source)
get_logger().debug("copied from container", source=source, destination=str(destination))
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal even if running.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
if force and container.status in ("running", "paused"):
container.kill()
container.remove()
get_logger().debug("removed container", container_id=identifier)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting for it to complete.
:param identifier: Container identifier.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
container.start()
get_logger().debug("started container (detached)", container_id=identifier)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status (e.g., "running", "exited", "created").
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
return str(container.status)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container gracefully.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
if container.status == "running":
container.stop(timeout=timeout)
get_logger().debug("stopped container", container_id=identifier)
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a running container using exec.
:param identifier: Container identifier.
:param path: Path to file inside container.
:returns: File contents as string.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
(status, (stdout, stderr)) = container.exec_run(cmd=["cat", path], demux=True)
if status != 0:
error_msg = stderr.decode("utf-8", errors="replace") if stderr else "File not found"
get_logger().debug("failed to read file from container", path=path, error=error_msg)
return ""
return stdout.decode("utf-8", errors="replace") if stdout else ""
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
client: PodmanClient = self.get_client()
with client:
containers = client.containers.list(all=all_containers)
return [
{
"Id": str(c.id),
"Names": [c.name] if hasattr(c, "name") else [],
"Status": str(c.status),
"Image": str(c.image) if hasattr(c, "image") else "",
}
for c in containers
]

View File

@@ -0,0 +1,19 @@
"""FuzzForge storage abstractions.
Storage class requires boto3. Import it explicitly:
from fuzzforge_common.storage.s3 import Storage
"""
from fuzzforge_common.storage.exceptions import (
FuzzForgeStorageError,
StorageConnectionError,
StorageDownloadError,
StorageUploadError,
)
__all__ = [
"FuzzForgeStorageError",
"StorageConnectionError",
"StorageDownloadError",
"StorageUploadError",
]

View File

@@ -0,0 +1,20 @@
from pydantic import BaseModel
from fuzzforge_common.storage.s3 import Storage
class StorageConfiguration(BaseModel):
"""TODO."""
#: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO).
endpoint: str
#: S3 access key ID for authentication.
access_key: str
#: S3 secret access key for authentication.
secret_key: str
def into_storage(self) -> Storage:
"""TODO."""
return Storage(endpoint=self.endpoint, access_key=self.access_key, secret_key=self.secret_key)

View File

@@ -0,0 +1,108 @@
from fuzzforge_common.exceptions import FuzzForgeError
class FuzzForgeStorageError(FuzzForgeError):
"""Base exception for all storage-related errors.
Raised when storage operations (upload, download, connection) fail
during workflow execution.
"""
class StorageConnectionError(FuzzForgeStorageError):
"""Failed to connect to storage service.
:param endpoint: The storage endpoint that failed to connect.
:param reason: The underlying exception message.
"""
def __init__(self, endpoint: str, reason: str) -> None:
"""Initialize storage connection error.
:param endpoint: The storage endpoint that failed to connect.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to connect to storage at {endpoint}: {reason}",
)
self.endpoint = endpoint
self.reason = reason
class StorageUploadError(FuzzForgeStorageError):
"""Failed to upload object to storage.
:param bucket: The target bucket name.
:param object_key: The target object key.
:param reason: The underlying exception message.
"""
def __init__(self, bucket: str, object_key: str, reason: str) -> None:
"""Initialize storage upload error.
:param bucket: The target bucket name.
:param object_key: The target object key.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to upload to {bucket}/{object_key}: {reason}",
)
self.bucket = bucket
self.object_key = object_key
self.reason = reason
class StorageDownloadError(FuzzForgeStorageError):
"""Failed to download object from storage.
:param bucket: The source bucket name.
:param object_key: The source object key.
:param reason: The underlying exception message.
"""
def __init__(self, bucket: str, object_key: str, reason: str) -> None:
"""Initialize storage download error.
:param bucket: The source bucket name.
:param object_key: The source object key.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to download from {bucket}/{object_key}: {reason}",
)
self.bucket = bucket
self.object_key = object_key
self.reason = reason
class StorageDeletionError(FuzzForgeStorageError):
"""Failed to delete bucket from storage.
:param bucket: The bucket name that failed to delete.
:param reason: The underlying exception message.
"""
def __init__(self, bucket: str, reason: str) -> None:
"""Initialize storage deletion error.
:param bucket: The bucket name that failed to delete.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to delete bucket {bucket}: {reason}",
)
self.bucket = bucket
self.reason = reason

View File

@@ -0,0 +1,351 @@
from __future__ import annotations
from pathlib import Path, PurePath
from tarfile import TarInfo
from tarfile import open as Archive # noqa: N812
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any, cast
from botocore.exceptions import ClientError
from fuzzforge_common.storage.exceptions import StorageDeletionError, StorageDownloadError, StorageUploadError
if TYPE_CHECKING:
from botocore.client import BaseClient
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structlog logger instance.
Uses deferred import pattern required by Temporal for serialization.
:returns: Configured structlog logger.
"""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
class Storage:
"""S3-compatible storage backend implementation using boto3.
Supports MinIO, AWS S3, and other S3-compatible storage services.
Uses error-driven approach (EAFP) to handle bucket creation and
avoid race conditions.
"""
#: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO).
__endpoint: str
#: S3 access key ID for authentication.
__access_key: str
#: S3 secret access key for authentication.
__secret_key: str
def __init__(self, endpoint: str, access_key: str, secret_key: str) -> None:
"""Initialize an instance of the class.
:param endpoint: TODO.
:param access_key: TODO.
:param secret_key: TODO.
"""
self.__endpoint = endpoint
self.__access_key = access_key
self.__secret_key = secret_key
def _get_client(self) -> BaseClient:
"""Create boto3 S3 client with configured credentials.
Uses deferred import pattern required by Temporal for serialization.
:returns: Configured boto3 S3 client.
"""
import boto3 # noqa: PLC0415 (required by temporal)
return boto3.client(
"s3",
endpoint_url=self.__endpoint,
aws_access_key_id=self.__access_key,
aws_secret_access_key=self.__secret_key,
)
def create_bucket(self, bucket: str) -> None:
"""Create the S3 bucket if it does not already exist.
Idempotent operation - succeeds if bucket already exists and is owned by you.
Fails if bucket exists but is owned by another account.
:raise ClientError: If bucket creation fails (permissions, name conflicts, etc.).
"""
logger = get_logger()
client = self._get_client()
logger.debug("creating_bucket", bucket=bucket)
try:
client.create_bucket(Bucket=bucket)
logger.info("bucket_created", bucket=bucket)
except ClientError as e:
error_code = e.response.get("Error", {}).get("Code")
# Bucket already exists and we own it - this is fine
if error_code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
logger.debug(
"bucket_already_exists",
bucket=bucket,
error_code=error_code,
)
return
# Other errors are actual failures
logger.exception(
"bucket_creation_failed",
bucket=bucket,
error_code=error_code,
)
raise
def delete_bucket(self, bucket: str) -> None:
"""Delete an S3 bucket and all its contents.
Idempotent operation - succeeds if bucket doesn't exist.
Handles pagination for buckets with many objects.
:param bucket: The name of the bucket to delete.
:raises StorageDeletionError: If bucket deletion fails.
"""
logger = get_logger()
client = self._get_client()
logger.debug("deleting_bucket", bucket=bucket)
try:
# S3 requires bucket to be empty before deletion
# Delete all objects first with pagination support
continuation_token = None
while True:
# List objects (up to 1000 per request)
list_params = {"Bucket": bucket}
if continuation_token:
list_params["ContinuationToken"] = continuation_token
response = client.list_objects_v2(**list_params)
# Delete objects if any exist (max 1000 per delete_objects call)
if "Contents" in response:
objects = [{"Key": obj["Key"]} for obj in response["Contents"]]
client.delete_objects(Bucket=bucket, Delete={"Objects": objects})
logger.debug("deleted_objects", bucket=bucket, count=len(objects))
# Check if more objects exist
if not response.get("IsTruncated", False):
break
continuation_token = response.get("NextContinuationToken")
# Now delete the empty bucket
client.delete_bucket(Bucket=bucket)
logger.info("bucket_deleted", bucket=bucket)
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
# Idempotent - bucket already doesn't exist
if error_code == "NoSuchBucket":
logger.debug("bucket_does_not_exist", bucket=bucket)
return
# Other errors are actual failures
logger.exception(
"bucket_deletion_failed",
bucket=bucket,
error_code=error_code,
)
raise StorageDeletionError(bucket=bucket, reason=str(error)) from error
def upload_file(
self,
bucket: str,
file: Path,
key: str,
) -> None:
"""Upload archive file to S3 storage at specified object key.
Assumes bucket exists. Fails gracefully if bucket or other resources missing.
:param bucket: TODO.
:param file: Local path to the archive file to upload.
:param key: Object key (path) in S3 where file should be uploaded.
:raise StorageUploadError: If upload operation fails.
"""
from boto3.exceptions import S3UploadFailedError # noqa: PLC0415 (required by 'temporal' at runtime)
logger = get_logger()
client = self._get_client()
logger.debug(
"uploading_archive_to_storage",
bucket=bucket,
object_key=key,
archive_path=str(file),
)
try:
client.upload_file(
Filename=str(file),
Bucket=bucket,
Key=key,
)
logger.info(
"archive_uploaded_successfully",
bucket=bucket,
object_key=key,
)
except S3UploadFailedError as e:
# Check if this is a NoSuchBucket error - create bucket and retry
if "NoSuchBucket" in str(e):
logger.info(
"bucket_does_not_exist_creating",
bucket=bucket,
)
self.create_bucket(bucket=bucket)
# Retry upload after creating bucket
try:
client.upload_file(
Filename=str(file),
Bucket=bucket,
Key=key,
)
logger.info(
"archive_uploaded_successfully_after_bucket_creation",
bucket=bucket,
object_key=key,
)
except S3UploadFailedError as retry_error:
logger.exception(
"upload_failed_after_bucket_creation",
bucket=bucket,
object_key=key,
)
raise StorageUploadError(
bucket=bucket,
object_key=key,
reason=str(retry_error),
) from retry_error
else:
logger.exception(
"upload_failed",
bucket=bucket,
object_key=key,
)
raise StorageUploadError(
bucket=bucket,
object_key=key,
reason=str(e),
) from e
def download_file(self, bucket: str, key: PurePath) -> Path:
"""Download a single file from S3 storage.
Downloads the file to a temporary location and returns the path.
:param bucket: S3 bucket name.
:param key: Object key (path) in S3 to download.
:returns: Path to the downloaded file.
:raise StorageDownloadError: If download operation fails.
"""
logger = get_logger()
client = self._get_client()
logger.debug(
"downloading_file_from_storage",
bucket=bucket,
object_key=str(key),
)
try:
# Create temporary file for download
with NamedTemporaryFile(delete=False, suffix=".tar.gz") as temp_file:
temp_path = Path(temp_file.name)
# Download object to temp file
client.download_file(
Bucket=bucket,
Key=str(key),
Filename=str(temp_path),
)
logger.info(
"file_downloaded_successfully",
bucket=bucket,
object_key=str(key),
local_path=str(temp_path),
)
return temp_path
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
logger.exception(
"download_failed",
bucket=bucket,
object_key=str(key),
error_code=error_code,
)
raise StorageDownloadError(
bucket=bucket,
object_key=str(key),
reason=f"{error_code}: {error!s}",
) from error
def download_directory(self, bucket: str, directory: PurePath) -> Path:
"""TODO.
:param bucket: TODO.
:param directory: TODO.
:returns: TODO.
"""
with NamedTemporaryFile(delete=False) as file:
path: Path = Path(file.name)
# end-with
client: Any = self._get_client()
with Archive(name=str(path), mode="w:gz") as archive:
paginator = client.get_paginator("list_objects_v2")
try:
pages = paginator.paginate(Bucket=bucket, Prefix=str(directory))
except ClientError as exception:
raise StorageDownloadError(
bucket=bucket,
object_key=str(directory),
reason=exception.response["Error"]["Code"],
) from exception
for page in pages:
for entry in page.get("Contents", []):
key: str = entry["Key"]
try:
response: dict[str, Any] = client.get_object(Bucket=bucket, Key=key)
except ClientError as exception:
raise StorageDownloadError(
bucket=bucket,
object_key=key,
reason=exception.response["Error"]["Code"],
) from exception
archive.addfile(TarInfo(name=key), fileobj=response["Body"])
# end-for
# end-for
# end-with
return path

View File

@@ -0,0 +1,8 @@
from enum import StrEnum
class TemporalQueues(StrEnum):
"""Enumeration of available `Temporal Task Queues`."""
#: The default task queue.
DEFAULT = "default-task-queue"

View File

@@ -0,0 +1,46 @@
from enum import StrEnum
from typing import Literal
from fuzzforge_types import FuzzForgeWorkflowIdentifier # noqa: TC002 (required by 'pydantic' at runtime)
from pydantic import BaseModel
class Base(BaseModel):
"""TODO."""
class FuzzForgeWorkflowSteps(StrEnum):
"""Workflow step types."""
#: Execute a FuzzForge module
RUN_FUZZFORGE_MODULE = "run-fuzzforge-module"
class FuzzForgeWorkflowStep(Base):
"""TODO."""
#: The type of the workflow's step.
kind: FuzzForgeWorkflowSteps
class RunFuzzForgeModule(FuzzForgeWorkflowStep):
"""Execute a FuzzForge module."""
kind: Literal[FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE] = FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE
#: The name of the module.
module: str
#: The container of the module.
container: str
class FuzzForgeWorkflowDefinition(Base):
"""The definition of a FuzzForge workflow."""
#: The author of the workflow.
author: str
#: The identifier of the workflow.
identifier: FuzzForgeWorkflowIdentifier
#: The name of the workflow.
name: str
#: The collection of steps that compose the workflow.
steps: list[RunFuzzForgeModule]

View File

@@ -0,0 +1,24 @@
from pydantic import BaseModel
from fuzzforge_common.sandboxes.engines.docker.configuration import (
DockerConfiguration, # noqa: TC001 (required by pydantic at runtime)
)
from fuzzforge_common.sandboxes.engines.podman.configuration import (
PodmanConfiguration, # noqa: TC001 (required by pydantic at runtime)
)
from fuzzforge_common.storage.configuration import StorageConfiguration # noqa: TC001 (required by pydantic at runtime)
class TemporalWorkflowParameters(BaseModel):
"""Base parameters for Temporal workflows.
Provides common configuration shared across all workflow types,
including sandbox engine and storage backend instances.
"""
#: Sandbox engine for container operations (Docker or Podman).
engine_configuration: PodmanConfiguration | DockerConfiguration
#: Storage backend for uploading/downloading execution artifacts.
storage_configuration: StorageConfiguration

View File

@@ -0,0 +1,108 @@
"""Helper utilities for working with bridge transformations."""
from pathlib import Path
from typing import Any
def load_transform_from_file(file_path: str | Path) -> str:
"""Load bridge transformation code from a Python file.
This reads the transformation function from a .py file and extracts
the code as a string suitable for the bridge module.
Args:
file_path: Path to Python file containing transform() function
Returns:
Python code as a string
Example:
>>> code = load_transform_from_file("transformations/add_line_numbers.py")
>>> # code contains the transform() function as a string
"""
path = Path(file_path)
if not path.exists():
raise FileNotFoundError(f"Transformation file not found: {file_path}")
if path.suffix != ".py":
raise ValueError(f"Transformation file must be .py file, got: {path.suffix}")
# Read the entire file
code = path.read_text()
return code
def create_bridge_input(
transform_file: str | Path,
input_filename: str | None = None,
output_filename: str | None = None,
) -> dict[str, Any]:
"""Create bridge module input configuration from a transformation file.
Args:
transform_file: Path to Python file with transform() function
input_filename: Optional specific input file to transform
output_filename: Optional specific output filename
Returns:
Dictionary suitable for bridge module's input.json
Example:
>>> config = create_bridge_input("transformations/add_line_numbers.py")
>>> import json
>>> json.dump(config, open("input.json", "w"))
"""
code = load_transform_from_file(transform_file)
return {
"code": code,
"input_filename": input_filename,
"output_filename": output_filename,
}
def validate_transform_function(file_path: str | Path) -> bool:
"""Validate that a Python file contains a valid transform() function.
Args:
file_path: Path to Python file to validate
Returns:
True if valid, raises exception otherwise
Raises:
ValueError: If transform() function is not found or invalid
"""
code = load_transform_from_file(file_path)
# Check if transform function is defined
if "def transform(" not in code:
raise ValueError(
f"File {file_path} must contain a 'def transform(data)' function"
)
# Try to compile the code
try:
compile(code, str(file_path), "exec")
except SyntaxError as e:
raise ValueError(f"Syntax error in {file_path}: {e}") from e
# Try to execute and verify transform exists
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(code, namespace)
except Exception as e:
raise ValueError(f"Failed to execute {file_path}: {e}") from e
if "transform" not in namespace:
raise ValueError(f"No 'transform' function found in {file_path}")
if not callable(namespace["transform"]):
raise ValueError(f"'transform' in {file_path} is not callable")
return True

View File

@@ -0,0 +1,27 @@
from fuzzforge_types import (
FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime)
FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime)
)
from fuzzforge_common.workflows.base.definitions import (
FuzzForgeWorkflowDefinition, # noqa: TC001 (required by pydantic at runtime)
)
from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters
class ExecuteFuzzForgeWorkflowParameters(TemporalWorkflowParameters):
"""Parameters for the default FuzzForge workflow orchestration.
Contains workflow definition and execution tracking identifiers
for coordinating multi-module workflows.
"""
#: UUID7 identifier of this specific workflow execution.
execution_identifier: FuzzForgeExecutionIdentifier
#: UUID7 identifier of the project this execution belongs to.
project_identifier: FuzzForgeProjectIdentifier
#: The definition of the FuzzForge workflow to run.
workflow_definition: FuzzForgeWorkflowDefinition

View File

@@ -0,0 +1,80 @@
from typing import Any, Literal
from fuzzforge_types import (
FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime)
FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime)
)
from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters
class ExecuteFuzzForgeModuleParameters(TemporalWorkflowParameters):
"""Parameters for executing a single FuzzForge module workflow.
Contains module execution configuration including container image,
project context, and execution tracking identifiers.
Supports workflow chaining where modules can be executed in sequence,
with each module's output becoming the next module's input.
"""
#: The identifier of this module execution.
execution_identifier: FuzzForgeExecutionIdentifier
#: The identifier/name of the module to execute.
#: FIXME: Currently accepts both UUID (for registry lookups) and container names (e.g., "text-generator:0.0.1").
#: This should be split into module_identifier (UUID) and container_image (string) in the future.
module_identifier: str
#: The identifier of the project this module execution belongs to.
project_identifier: FuzzForgeProjectIdentifier
#: Optional configuration dictionary for the module.
#: Will be written to /data/input/config.json in the sandbox.
module_configuration: dict[str, Any] | None = None
# Workflow chaining fields
#: The identifier of the parent workflow execution (if part of a multi-module workflow).
#: For standalone module executions, this equals execution_identifier.
workflow_execution_identifier: FuzzForgeExecutionIdentifier | None = None
#: Position of this module in the workflow (0-based).
#: 0 = first module (reads from project assets)
#: N > 0 = subsequent module (reads from previous module's output)
step_index: int = 0
#: Execution identifier of the previous module in the workflow chain.
#: None for first module (step_index=0).
#: Used to locate previous module's output in storage.
previous_step_execution_identifier: FuzzForgeExecutionIdentifier | None = None
class WorkflowStep(TemporalWorkflowParameters):
"""A step in a workflow - a module execution.
Steps are executed sequentially in a workflow. Each step runs a containerized module.
Examples:
# Module step
WorkflowStep(
step_index=0,
step_type="module",
module_identifier="text-generator:0.0.1"
)
"""
#: Position of this step in the workflow (0-based)
step_index: int
#: Type of step: "module" (bridges are also modules now)
step_type: Literal["module"]
#: Module identifier (container image name like "text-generator:0.0.1")
#: Required if step_type="module"
module_identifier: str | None = None
#: Optional module configuration
module_configuration: dict[str, Any] | None = None

View File

View File

View File

@@ -0,0 +1 @@
pytest_plugins = ["fuzzforge_tests.fixtures"]

View File

View File

@@ -0,0 +1,9 @@
import pytest
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
@pytest.fixture
def podman_engine(podman_socket: str) -> Podman:
"""TODO."""
return Podman(socket=podman_socket)

View File

@@ -0,0 +1,21 @@
from typing import TYPE_CHECKING
from uuid import uuid4
if TYPE_CHECKING:
from pathlib import Path
from podman import PodmanClient
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
def test_can_register_oci(
path_to_oci: Path,
podman_engine: Podman,
podman_client: PodmanClient,
) -> None:
"""TODO."""
repository: str = str(uuid4())
podman_engine.register_archive(archive=path_to_oci, repository=repository)
assert podman_client.images.exists(key=repository)
podman_client.images.get(name=repository).remove()

View File

@@ -0,0 +1,42 @@
from pathlib import Path
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from fuzzforge_common.storage.configuration import StorageConfiguration
def test_download_directory(
storage_configuration: StorageConfiguration,
boto3_client: Any,
random_bucket: str,
tmp_path: Path,
) -> None:
"""TODO."""
bucket = random_bucket
storage = storage_configuration.into_storage()
d1 = tmp_path.joinpath("d1")
f1 = d1.joinpath("f1")
d2 = tmp_path.joinpath("d2")
f2 = d2.joinpath("f2")
d3 = d2.joinpath("d3")
f3 = d3.joinpath("d3")
d1.mkdir()
d2.mkdir()
d3.mkdir()
f1.touch()
f2.touch()
f3.touch()
for path in [f1, f2, f3]:
key: Path = Path("assets", path.relative_to(other=tmp_path))
boto3_client.upload_file(
Bucket=bucket,
Filename=str(path),
Key=str(key),
)
path = storage.download_directory(bucket=bucket, directory="assets")
assert path.is_file()