mirror of
https://github.com/FuzzingLabs/fuzzforge_ai.git
synced 2026-02-12 21:12:56 +00:00
feat: FuzzForge AI - complete rewrite for OSS release
This commit is contained in:
42
fuzzforge-common/Makefile
Normal file
42
fuzzforge-common/Makefile
Normal file
@@ -0,0 +1,42 @@
|
||||
PACKAGE=$(word 1, $(shell uv version))
|
||||
VERSION=$(word 2, $(shell uv version))
|
||||
|
||||
ARTIFACTS?=./dist
|
||||
|
||||
SOURCES=./src
|
||||
TESTS=./tests
|
||||
|
||||
.PHONY: bandit clean format mypy pytest ruff version wheel
|
||||
|
||||
bandit:
|
||||
uv run bandit --recursive $(SOURCES)
|
||||
|
||||
clean:
|
||||
@find . -type d \( \
|
||||
-name '*.egg-info' \
|
||||
-o -name '.mypy_cache' \
|
||||
-o -name '.pytest_cache' \
|
||||
-o -name '.ruff_cache' \
|
||||
-o -name '__pycache__' \
|
||||
\) -printf 'removing directory %p\n' -exec rm -rf {} +
|
||||
|
||||
cloc:
|
||||
cloc $(SOURCES)
|
||||
|
||||
format:
|
||||
uv run ruff format $(SOURCES) $(TESTS)
|
||||
|
||||
mypy:
|
||||
uv run mypy $(SOURCES)
|
||||
|
||||
pytest:
|
||||
uv run pytest $(TESTS)
|
||||
|
||||
ruff:
|
||||
uv run ruff check --fix $(SOURCES) $(TESTS)
|
||||
|
||||
version:
|
||||
@echo '$(PACKAGE)@$(VERSION)'
|
||||
|
||||
wheel:
|
||||
uv build --out-dir $(ARTIFACTS)
|
||||
3
fuzzforge-common/README.md
Normal file
3
fuzzforge-common/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# FuzzForge Common
|
||||
|
||||
...
|
||||
12
fuzzforge-common/mypy.ini
Normal file
12
fuzzforge-common/mypy.ini
Normal file
@@ -0,0 +1,12 @@
|
||||
[mypy]
|
||||
plugins = pydantic.mypy
|
||||
strict = True
|
||||
warn_unused_ignores = True
|
||||
warn_redundant_casts = True
|
||||
warn_return_any = True
|
||||
|
||||
[mypy-botocore.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-boto3.*]
|
||||
ignore_missing_imports = True
|
||||
25
fuzzforge-common/pyproject.toml
Normal file
25
fuzzforge-common/pyproject.toml
Normal file
@@ -0,0 +1,25 @@
|
||||
[project]
|
||||
name = "fuzzforge-common"
|
||||
version = "0.0.1"
|
||||
description = "FuzzForge's common types and utilities."
|
||||
authors = []
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"fuzzforge-types==0.0.1",
|
||||
"podman==5.6.0",
|
||||
"pydantic==2.12.4",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
lints = [
|
||||
"bandit==1.8.6",
|
||||
"mypy==1.18.2",
|
||||
"ruff==0.14.4",
|
||||
]
|
||||
tests = [
|
||||
"pytest==9.0.2",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
fuzzforge-types = { workspace = true }
|
||||
3
fuzzforge-common/pytest.ini
Normal file
3
fuzzforge-common/pytest.ini
Normal file
@@ -0,0 +1,3 @@
|
||||
[pytest]
|
||||
env =
|
||||
DOCKER_HOST=unix:///run/user/1000/podman/podman.sock
|
||||
20
fuzzforge-common/ruff.toml
Normal file
20
fuzzforge-common/ruff.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
line-length = 120
|
||||
|
||||
[lint]
|
||||
select = [ "ALL" ]
|
||||
ignore = [
|
||||
"COM812", # conflicts with the formatter
|
||||
"D100", # ignoring missing docstrings in public modules
|
||||
"D104", # ignoring missing docstrings in public packages
|
||||
"D203", # conflicts with 'D211'
|
||||
"D213", # conflicts with 'D212'
|
||||
"TD002", # ignoring missing author in 'TODO' statements
|
||||
"TD003", # ignoring missing issue link in 'TODO' statements
|
||||
]
|
||||
|
||||
[lint.per-file-ignores]
|
||||
"tests/*" = [
|
||||
"ANN401", # allowing 'typing.Any' to be used to type function parameters in tests
|
||||
"PLR2004", # allowing comparisons using unamed numerical constants in tests
|
||||
"S101", # allowing 'assert' statements in tests
|
||||
]
|
||||
54
fuzzforge-common/src/fuzzforge_common/__init__.py
Normal file
54
fuzzforge-common/src/fuzzforge_common/__init__.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""FuzzForge Common - Shared abstractions and implementations for FuzzForge.
|
||||
|
||||
This package provides:
|
||||
- Sandbox engine abstractions (Podman, Docker)
|
||||
- Storage abstractions (S3) - requires 'storage' extra
|
||||
- Common exceptions
|
||||
|
||||
Example usage:
|
||||
from fuzzforge_common import (
|
||||
AbstractFuzzForgeSandboxEngine,
|
||||
ImageInfo,
|
||||
Podman,
|
||||
PodmanConfiguration,
|
||||
)
|
||||
|
||||
# For storage (requires boto3):
|
||||
from fuzzforge_common.storage import Storage
|
||||
"""
|
||||
|
||||
from fuzzforge_common.exceptions import FuzzForgeError
|
||||
from fuzzforge_common.sandboxes import (
|
||||
AbstractFuzzForgeEngineConfiguration,
|
||||
AbstractFuzzForgeSandboxEngine,
|
||||
Docker,
|
||||
DockerConfiguration,
|
||||
FuzzForgeSandboxEngines,
|
||||
ImageInfo,
|
||||
Podman,
|
||||
PodmanConfiguration,
|
||||
)
|
||||
|
||||
# Storage exceptions are always available (no boto3 required)
|
||||
from fuzzforge_common.storage.exceptions import (
|
||||
FuzzForgeStorageError,
|
||||
StorageConnectionError,
|
||||
StorageDownloadError,
|
||||
StorageUploadError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AbstractFuzzForgeEngineConfiguration",
|
||||
"AbstractFuzzForgeSandboxEngine",
|
||||
"Docker",
|
||||
"DockerConfiguration",
|
||||
"FuzzForgeError",
|
||||
"FuzzForgeSandboxEngines",
|
||||
"FuzzForgeStorageError",
|
||||
"ImageInfo",
|
||||
"Podman",
|
||||
"PodmanConfiguration",
|
||||
"StorageConnectionError",
|
||||
"StorageDownloadError",
|
||||
"StorageUploadError",
|
||||
]
|
||||
24
fuzzforge-common/src/fuzzforge_common/exceptions.py
Normal file
24
fuzzforge-common/src/fuzzforge_common/exceptions.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any
|
||||
|
||||
|
||||
class FuzzForgeError(Exception):
|
||||
"""Base exception for all FuzzForge custom exceptions.
|
||||
|
||||
All domain exceptions should inherit from this base to enable
|
||||
consistent exception handling and hierarchy navigation.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
|
||||
"""Initialize FuzzForge error.
|
||||
|
||||
:param message: Error message.
|
||||
:param details: Optional error details dictionary.
|
||||
|
||||
"""
|
||||
Exception.__init__(self, message)
|
||||
self.message = message
|
||||
self.details = details or {}
|
||||
0
fuzzforge-common/src/fuzzforge_common/py.typed
Normal file
0
fuzzforge-common/src/fuzzforge_common/py.typed
Normal file
23
fuzzforge-common/src/fuzzforge_common/sandboxes/__init__.py
Normal file
23
fuzzforge-common/src/fuzzforge_common/sandboxes/__init__.py
Normal file
@@ -0,0 +1,23 @@
|
||||
"""FuzzForge sandbox abstractions and implementations."""
|
||||
|
||||
from fuzzforge_common.sandboxes.engines import (
|
||||
AbstractFuzzForgeEngineConfiguration,
|
||||
AbstractFuzzForgeSandboxEngine,
|
||||
Docker,
|
||||
DockerConfiguration,
|
||||
FuzzForgeSandboxEngines,
|
||||
ImageInfo,
|
||||
Podman,
|
||||
PodmanConfiguration,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AbstractFuzzForgeEngineConfiguration",
|
||||
"AbstractFuzzForgeSandboxEngine",
|
||||
"Docker",
|
||||
"DockerConfiguration",
|
||||
"FuzzForgeSandboxEngines",
|
||||
"ImageInfo",
|
||||
"Podman",
|
||||
"PodmanConfiguration",
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Container engine implementations for FuzzForge sandboxes."""
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.base import (
|
||||
AbstractFuzzForgeEngineConfiguration,
|
||||
AbstractFuzzForgeSandboxEngine,
|
||||
ImageInfo,
|
||||
)
|
||||
from fuzzforge_common.sandboxes.engines.docker import Docker, DockerConfiguration
|
||||
from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines
|
||||
from fuzzforge_common.sandboxes.engines.podman import Podman, PodmanConfiguration
|
||||
|
||||
__all__ = [
|
||||
"AbstractFuzzForgeEngineConfiguration",
|
||||
"AbstractFuzzForgeSandboxEngine",
|
||||
"Docker",
|
||||
"DockerConfiguration",
|
||||
"FuzzForgeSandboxEngines",
|
||||
"ImageInfo",
|
||||
"Podman",
|
||||
"PodmanConfiguration",
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Base engine abstractions."""
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.base.configuration import (
|
||||
AbstractFuzzForgeEngineConfiguration,
|
||||
)
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import (
|
||||
AbstractFuzzForgeSandboxEngine,
|
||||
ImageInfo,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AbstractFuzzForgeEngineConfiguration",
|
||||
"AbstractFuzzForgeSandboxEngine",
|
||||
"ImageInfo",
|
||||
]
|
||||
@@ -0,0 +1,24 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.enumeration import (
|
||||
FuzzForgeSandboxEngines, # noqa: TC001 (required by 'pydantic' at runtime)
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine
|
||||
|
||||
|
||||
class AbstractFuzzForgeEngineConfiguration(ABC, BaseModel):
|
||||
"""TODO."""
|
||||
|
||||
#: TODO.
|
||||
kind: FuzzForgeSandboxEngines
|
||||
|
||||
@abstractmethod
|
||||
def into_engine(self) -> AbstractFuzzForgeSandboxEngine:
|
||||
"""TODO."""
|
||||
message: str = f"method 'into_engine' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
@@ -0,0 +1,281 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path, PurePath
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageInfo:
|
||||
"""Information about a container image."""
|
||||
|
||||
#: Full image reference (e.g., "localhost/fuzzforge-module-echidna:latest").
|
||||
reference: str
|
||||
|
||||
#: Repository name (e.g., "localhost/fuzzforge-module-echidna").
|
||||
repository: str
|
||||
|
||||
#: Image tag (e.g., "latest").
|
||||
tag: str
|
||||
|
||||
#: Image ID (short hash).
|
||||
image_id: str | None = None
|
||||
|
||||
#: Image size in bytes.
|
||||
size: int | None = None
|
||||
|
||||
|
||||
class AbstractFuzzForgeSandboxEngine(ABC):
|
||||
"""Abstract class used as a base for all FuzzForge sandbox engine classes."""
|
||||
|
||||
@abstractmethod
|
||||
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
|
||||
"""List available container images.
|
||||
|
||||
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
|
||||
:returns: List of ImageInfo objects for available images.
|
||||
|
||||
"""
|
||||
message: str = f"method 'list_images' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def register_archive(self, archive: Path, repository: str) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param archive: TODO.
|
||||
|
||||
"""
|
||||
message: str = f"method 'register_archive' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def spawn_sandbox(self, image: str) -> str:
|
||||
"""Spawn a sandbox based on the given image.
|
||||
|
||||
:param image: The image the sandbox should be based on.
|
||||
:returns: The sandbox identifier.
|
||||
|
||||
"""
|
||||
message: str = f"method 'spawn_sandbox' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: TODO.
|
||||
:param source: TODO.
|
||||
:param destination: TODO.
|
||||
|
||||
"""
|
||||
message: str = f"method 'push_archive_to_sandbox' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def start_sandbox(self, identifier: str) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: The identifier of the sandbox to start.
|
||||
|
||||
"""
|
||||
message: str = f"method 'start_sandbox' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
|
||||
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
|
||||
|
||||
:param sandbox: The identifier of the sandbox.
|
||||
:param command: The command to run.
|
||||
|
||||
"""
|
||||
message: str = f"method 'execute_inside_sandbox' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: TODO.
|
||||
:param source: TODO.
|
||||
:returns: TODO.
|
||||
|
||||
"""
|
||||
message: str = f"method 'pull_archive_from_sandbox' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def terminate_sandbox(self, identifier: str) -> None:
|
||||
"""Terminate the sandbox matching the given identifier.
|
||||
|
||||
:param identifier: The identifier of the sandbox to terminate.
|
||||
|
||||
"""
|
||||
message: str = f"method 'terminate_sandbox' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Extended Container Operations
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def image_exists(self, image: str) -> bool:
|
||||
"""Check if a container image exists locally.
|
||||
|
||||
:param image: Full image reference (e.g., "localhost/module:latest").
|
||||
:returns: True if image exists, False otherwise.
|
||||
|
||||
"""
|
||||
message: str = f"method 'image_exists' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def pull_image(self, image: str, timeout: int = 300) -> None:
|
||||
"""Pull an image from a container registry.
|
||||
|
||||
:param image: Full image reference to pull.
|
||||
:param timeout: Timeout in seconds for the pull operation.
|
||||
:raises FuzzForgeError: If pull fails.
|
||||
|
||||
"""
|
||||
message: str = f"method 'pull_image' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def tag_image(self, source: str, target: str) -> None:
|
||||
"""Tag an image with a new name.
|
||||
|
||||
:param source: Source image reference.
|
||||
:param target: Target image reference.
|
||||
|
||||
"""
|
||||
message: str = f"method 'tag_image' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def create_container(
|
||||
self,
|
||||
image: str,
|
||||
volumes: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Create a container from an image.
|
||||
|
||||
:param image: Image to create container from.
|
||||
:param volumes: Optional volume mappings {host_path: container_path}.
|
||||
:returns: Container identifier.
|
||||
|
||||
"""
|
||||
message: str = f"method 'create_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def start_container_attached(
|
||||
self,
|
||||
identifier: str,
|
||||
timeout: int = 600,
|
||||
) -> tuple[int, str, str]:
|
||||
"""Start a container and wait for it to complete.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param timeout: Timeout in seconds for execution.
|
||||
:returns: Tuple of (exit_code, stdout, stderr).
|
||||
|
||||
"""
|
||||
message: str = f"method 'start_container_attached' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
|
||||
"""Copy a file or directory to a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path on host.
|
||||
:param destination: Destination path in container.
|
||||
|
||||
"""
|
||||
message: str = f"method 'copy_to_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
|
||||
"""Copy a file or directory from a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path in container.
|
||||
:param destination: Destination path on host.
|
||||
|
||||
"""
|
||||
message: str = f"method 'copy_from_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def remove_container(self, identifier: str, *, force: bool = False) -> None:
|
||||
"""Remove a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param force: Force removal even if running.
|
||||
|
||||
"""
|
||||
message: str = f"method 'remove_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Continuous/Background Execution Operations
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def start_container(self, identifier: str) -> None:
|
||||
"""Start a container without waiting for it to complete (detached mode).
|
||||
|
||||
:param identifier: Container identifier.
|
||||
|
||||
"""
|
||||
message: str = f"method 'start_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def get_container_status(self, identifier: str) -> str:
|
||||
"""Get the status of a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:returns: Container status (e.g., "running", "exited", "created").
|
||||
|
||||
"""
|
||||
message: str = f"method 'get_container_status' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def stop_container(self, identifier: str, timeout: int = 10) -> None:
|
||||
"""Stop a running container gracefully.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param timeout: Seconds to wait before killing.
|
||||
|
||||
"""
|
||||
message: str = f"method 'stop_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def read_file_from_container(self, identifier: str, path: str) -> str:
|
||||
"""Read a file from inside a running container using exec.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param path: Path to file inside container.
|
||||
:returns: File contents as string.
|
||||
|
||||
"""
|
||||
message: str = f"method 'read_file_from_container' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def list_containers(self, all_containers: bool = True) -> list[dict]:
|
||||
"""List containers.
|
||||
|
||||
:param all_containers: Include stopped containers.
|
||||
:returns: List of container info dicts.
|
||||
|
||||
"""
|
||||
message: str = f"method 'list_containers' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Docker container engine implementation."""
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.docker.configuration import (
|
||||
DockerConfiguration,
|
||||
)
|
||||
from fuzzforge_common.sandboxes.engines.docker.engine import Docker
|
||||
|
||||
__all__ = [
|
||||
"Docker",
|
||||
"DockerConfiguration",
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.base.configuration import AbstractFuzzForgeEngineConfiguration
|
||||
from fuzzforge_common.sandboxes.engines.docker.engine import Docker
|
||||
from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine
|
||||
|
||||
|
||||
class DockerConfiguration(AbstractFuzzForgeEngineConfiguration):
|
||||
"""TODO."""
|
||||
|
||||
#: TODO.
|
||||
kind: Literal[FuzzForgeSandboxEngines.DOCKER] = FuzzForgeSandboxEngines.DOCKER
|
||||
|
||||
#: TODO.
|
||||
socket: str
|
||||
|
||||
def into_engine(self) -> AbstractFuzzForgeSandboxEngine:
|
||||
"""TODO."""
|
||||
return Docker(socket=self.socket)
|
||||
@@ -0,0 +1,174 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path, PurePath
|
||||
|
||||
|
||||
class Docker(AbstractFuzzForgeSandboxEngine):
|
||||
"""TODO."""
|
||||
|
||||
#: TODO.
|
||||
__socket: str
|
||||
|
||||
def __init__(self, socket: str) -> None:
|
||||
"""Initialize an instance of the class.
|
||||
|
||||
:param socket: TODO.
|
||||
|
||||
"""
|
||||
super().__init__()
|
||||
self.__socket = socket
|
||||
|
||||
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
|
||||
"""List available container images.
|
||||
|
||||
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
|
||||
:returns: List of ImageInfo objects for available images.
|
||||
|
||||
"""
|
||||
# TODO: Implement Docker image listing
|
||||
message: str = "Docker engine list_images is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def register_archive(self, archive: Path, repository: str) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param archive: TODO.
|
||||
|
||||
"""
|
||||
return super().register_archive(archive=archive, repository=repository)
|
||||
|
||||
def spawn_sandbox(self, image: str) -> str:
|
||||
"""Spawn a sandbox based on the given image.
|
||||
|
||||
:param image: The image the sandbox should be based on.
|
||||
:returns: The sandbox identifier.
|
||||
|
||||
"""
|
||||
return super().spawn_sandbox(image)
|
||||
|
||||
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: TODO.
|
||||
:param source: TODO.
|
||||
:param destination: TODO.
|
||||
|
||||
"""
|
||||
super().push_archive_to_sandbox(identifier, source, destination)
|
||||
|
||||
def start_sandbox(self, identifier: str) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: The identifier of the sandbox to start.
|
||||
|
||||
"""
|
||||
super().start_sandbox(identifier)
|
||||
|
||||
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
|
||||
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
|
||||
|
||||
:param sandbox: The identifier of the sandbox.
|
||||
:param command: The command to run.
|
||||
|
||||
"""
|
||||
super().execute_inside_sandbox(identifier, command)
|
||||
|
||||
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: TODO.
|
||||
:param source: TODO.
|
||||
:returns: TODO.
|
||||
|
||||
"""
|
||||
return super().pull_archive_from_sandbox(identifier, source)
|
||||
|
||||
def terminate_sandbox(self, identifier: str) -> None:
|
||||
"""Terminate the sandbox matching the given identifier.
|
||||
|
||||
:param identifier: The identifier of the sandbox to terminate.
|
||||
|
||||
"""
|
||||
super().terminate_sandbox(identifier)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Extended Container Operations (stubs - not yet implemented)
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def image_exists(self, image: str) -> bool:
|
||||
"""Check if a container image exists locally."""
|
||||
message: str = "Docker engine image_exists is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def pull_image(self, image: str, timeout: int = 300) -> None:
|
||||
"""Pull an image from a container registry."""
|
||||
message: str = "Docker engine pull_image is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def tag_image(self, source: str, target: str) -> None:
|
||||
"""Tag an image with a new name."""
|
||||
message: str = "Docker engine tag_image is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def create_container(
|
||||
self,
|
||||
image: str,
|
||||
volumes: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Create a container from an image."""
|
||||
message: str = "Docker engine create_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def start_container_attached(
|
||||
self,
|
||||
identifier: str,
|
||||
timeout: int = 600,
|
||||
) -> tuple[int, str, str]:
|
||||
"""Start a container and wait for it to complete."""
|
||||
message: str = "Docker engine start_container_attached is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
|
||||
"""Copy a file or directory to a container."""
|
||||
message: str = "Docker engine copy_to_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
|
||||
"""Copy a file or directory from a container."""
|
||||
message: str = "Docker engine copy_from_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def remove_container(self, identifier: str, *, force: bool = False) -> None:
|
||||
"""Remove a container."""
|
||||
message: str = "Docker engine remove_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def start_container(self, identifier: str) -> None:
|
||||
"""Start a container without waiting for it to complete."""
|
||||
message: str = "Docker engine start_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def get_container_status(self, identifier: str) -> str:
|
||||
"""Get the status of a container."""
|
||||
message: str = "Docker engine get_container_status is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def stop_container(self, identifier: str, timeout: int = 10) -> None:
|
||||
"""Stop a running container gracefully."""
|
||||
message: str = "Docker engine stop_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def read_file_from_container(self, identifier: str, path: str) -> str:
|
||||
"""Read a file from inside a running container using exec."""
|
||||
message: str = "Docker engine read_file_from_container is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
def list_containers(self, all_containers: bool = True) -> list[dict]:
|
||||
"""List containers."""
|
||||
message: str = "Docker engine list_containers is not yet implemented"
|
||||
raise NotImplementedError(message)
|
||||
@@ -0,0 +1,11 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class FuzzForgeSandboxEngines(StrEnum):
|
||||
"""TODO."""
|
||||
|
||||
#: TODO.
|
||||
DOCKER = "docker"
|
||||
|
||||
#: TODO.
|
||||
PODMAN = "podman"
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Podman container engine implementation."""
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.podman.cli import PodmanCLI
|
||||
from fuzzforge_common.sandboxes.engines.podman.configuration import (
|
||||
PodmanConfiguration,
|
||||
)
|
||||
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
|
||||
|
||||
__all__ = [
|
||||
"Podman",
|
||||
"PodmanCLI",
|
||||
"PodmanConfiguration",
|
||||
]
|
||||
@@ -0,0 +1,444 @@
|
||||
"""Podman CLI engine with custom storage support.
|
||||
|
||||
This engine uses subprocess calls to the Podman CLI instead of the socket API,
|
||||
allowing for custom storage paths (--root, --runroot) that work regardless of
|
||||
system Podman configuration or snap environment issues.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import tarfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path, PurePath
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from fuzzforge_common.exceptions import FuzzForgeError
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from structlog.stdlib import BoundLogger
|
||||
|
||||
|
||||
def get_logger() -> BoundLogger:
|
||||
"""Get structured logger."""
|
||||
from structlog import get_logger # noqa: PLC0415 (required by temporal)
|
||||
|
||||
return cast("BoundLogger", get_logger())
|
||||
|
||||
|
||||
class PodmanCLI(AbstractFuzzForgeSandboxEngine):
|
||||
"""Podman engine using CLI with custom storage paths.
|
||||
|
||||
This implementation uses subprocess calls to the Podman CLI with --root
|
||||
and --runroot flags, providing isolation from system Podman storage.
|
||||
This is particularly useful when running from VS Code snap which sets
|
||||
XDG_DATA_HOME to a version-specific path.
|
||||
"""
|
||||
|
||||
__graphroot: Path
|
||||
__runroot: Path
|
||||
|
||||
def __init__(self, graphroot: Path, runroot: Path) -> None:
|
||||
"""Initialize the PodmanCLI engine.
|
||||
|
||||
:param graphroot: Path to container image storage.
|
||||
:param runroot: Path to container runtime state.
|
||||
|
||||
"""
|
||||
AbstractFuzzForgeSandboxEngine.__init__(self)
|
||||
self.__graphroot = graphroot
|
||||
self.__runroot = runroot
|
||||
|
||||
# Ensure directories exist
|
||||
self.__graphroot.mkdir(parents=True, exist_ok=True)
|
||||
self.__runroot.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _base_cmd(self) -> list[str]:
|
||||
"""Get base Podman command with storage flags.
|
||||
|
||||
:returns: Base command list with --root and --runroot.
|
||||
|
||||
"""
|
||||
return [
|
||||
"podman",
|
||||
"--root", str(self.__graphroot),
|
||||
"--runroot", str(self.__runroot),
|
||||
]
|
||||
|
||||
def _run(self, args: list[str], *, check: bool = True, capture: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Run a Podman command.
|
||||
|
||||
:param args: Command arguments (without 'podman').
|
||||
:param check: Raise exception on non-zero exit.
|
||||
:param capture: Capture stdout/stderr.
|
||||
:returns: CompletedProcess result.
|
||||
|
||||
"""
|
||||
cmd = self._base_cmd() + args
|
||||
get_logger().debug("running podman command", cmd=" ".join(cmd))
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
check=check,
|
||||
capture_output=capture,
|
||||
text=True,
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Image Operations
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
|
||||
"""List available container images.
|
||||
|
||||
:param filter_prefix: Optional prefix to filter images.
|
||||
:returns: List of ImageInfo objects.
|
||||
|
||||
"""
|
||||
result = self._run(["images", "--format", "json"])
|
||||
images: list[ImageInfo] = []
|
||||
|
||||
try:
|
||||
data = json.loads(result.stdout) if result.stdout.strip() else []
|
||||
except json.JSONDecodeError:
|
||||
get_logger().warning("failed to parse podman images output")
|
||||
return images
|
||||
|
||||
for image in data:
|
||||
# Get repository and tag from Names
|
||||
names = image.get("Names") or []
|
||||
for name in names:
|
||||
if filter_prefix and not name.startswith(filter_prefix):
|
||||
continue
|
||||
|
||||
# Parse repository and tag
|
||||
if ":" in name:
|
||||
repo, tag = name.rsplit(":", 1)
|
||||
else:
|
||||
repo = name
|
||||
tag = "latest"
|
||||
|
||||
images.append(
|
||||
ImageInfo(
|
||||
reference=name,
|
||||
repository=repo,
|
||||
tag=tag,
|
||||
image_id=image.get("Id", "")[:12],
|
||||
size=image.get("Size"),
|
||||
)
|
||||
)
|
||||
|
||||
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
|
||||
return images
|
||||
|
||||
def image_exists(self, image: str) -> bool:
|
||||
"""Check if a container image exists locally.
|
||||
|
||||
:param image: Full image reference.
|
||||
:returns: True if image exists.
|
||||
|
||||
"""
|
||||
result = self._run(["image", "exists", image], check=False)
|
||||
return result.returncode == 0
|
||||
|
||||
def pull_image(self, image: str, timeout: int = 300) -> None:
|
||||
"""Pull an image from a container registry.
|
||||
|
||||
:param image: Full image reference.
|
||||
:param timeout: Timeout in seconds.
|
||||
|
||||
"""
|
||||
get_logger().info("pulling image", image=image)
|
||||
try:
|
||||
self._run(["pull", image])
|
||||
get_logger().info("image pulled successfully", image=image)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
message = f"Failed to pull image '{image}': {exc.stderr}"
|
||||
raise FuzzForgeError(message) from exc
|
||||
|
||||
def tag_image(self, source: str, target: str) -> None:
|
||||
"""Tag an image with a new name.
|
||||
|
||||
:param source: Source image reference.
|
||||
:param target: Target image reference.
|
||||
|
||||
"""
|
||||
self._run(["tag", source, target])
|
||||
get_logger().debug("tagged image", source=source, target=target)
|
||||
|
||||
def build_image(self, context_path: Path, tag: str, dockerfile: str = "Dockerfile") -> None:
|
||||
"""Build an image from a Dockerfile.
|
||||
|
||||
:param context_path: Path to build context.
|
||||
:param tag: Image tag.
|
||||
:param dockerfile: Dockerfile name.
|
||||
|
||||
"""
|
||||
get_logger().info("building image", tag=tag, context=str(context_path))
|
||||
self._run(["build", "-t", tag, "-f", dockerfile, str(context_path)])
|
||||
get_logger().info("image built successfully", tag=tag)
|
||||
|
||||
def register_archive(self, archive: Path, repository: str) -> None:
|
||||
"""Load an image from a tar archive.
|
||||
|
||||
:param archive: Path to tar archive.
|
||||
:param repository: Repository name for the loaded image.
|
||||
|
||||
"""
|
||||
result = self._run(["load", "-i", str(archive)])
|
||||
# Tag the loaded image
|
||||
# Parse loaded image ID from output
|
||||
for line in result.stdout.splitlines():
|
||||
if "Loaded image:" in line:
|
||||
loaded_image = line.split("Loaded image:")[-1].strip()
|
||||
self._run(["tag", loaded_image, f"{repository}:latest"])
|
||||
break
|
||||
get_logger().debug("registered archive", archive=str(archive), repository=repository)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Container Operations
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def spawn_sandbox(self, image: str) -> str:
|
||||
"""Spawn a sandbox (container) from an image.
|
||||
|
||||
:param image: Image to create container from.
|
||||
:returns: Container identifier.
|
||||
|
||||
"""
|
||||
result = self._run(["create", image])
|
||||
container_id = result.stdout.strip()
|
||||
get_logger().debug("created container", container_id=container_id)
|
||||
return container_id
|
||||
|
||||
def create_container(
|
||||
self,
|
||||
image: str,
|
||||
volumes: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Create a container from an image.
|
||||
|
||||
:param image: Image to create container from.
|
||||
:param volumes: Optional volume mappings {host_path: container_path}.
|
||||
:returns: Container identifier.
|
||||
|
||||
"""
|
||||
args = ["create"]
|
||||
if volumes:
|
||||
for host_path, container_path in volumes.items():
|
||||
args.extend(["-v", f"{host_path}:{container_path}:ro"])
|
||||
args.append(image)
|
||||
|
||||
result = self._run(args)
|
||||
container_id = result.stdout.strip()
|
||||
get_logger().debug("created container", container_id=container_id, image=image)
|
||||
return container_id
|
||||
|
||||
def start_sandbox(self, identifier: str) -> None:
|
||||
"""Start a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
|
||||
"""
|
||||
self._run(["start", identifier])
|
||||
get_logger().debug("started container", container_id=identifier)
|
||||
|
||||
def start_container(self, identifier: str) -> None:
|
||||
"""Start a container without waiting.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
|
||||
"""
|
||||
self._run(["start", identifier])
|
||||
get_logger().debug("started container (detached)", container_id=identifier)
|
||||
|
||||
def start_container_attached(
|
||||
self,
|
||||
identifier: str,
|
||||
timeout: int = 600,
|
||||
) -> tuple[int, str, str]:
|
||||
"""Start a container and wait for completion.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param timeout: Timeout in seconds.
|
||||
:returns: Tuple of (exit_code, stdout, stderr).
|
||||
|
||||
"""
|
||||
get_logger().debug("starting container attached", container_id=identifier)
|
||||
# Start the container
|
||||
self._run(["start", identifier])
|
||||
|
||||
# Wait for completion
|
||||
wait_result = self._run(["wait", identifier])
|
||||
exit_code = int(wait_result.stdout.strip()) if wait_result.stdout.strip() else -1
|
||||
|
||||
# Get logs
|
||||
stdout_result = self._run(["logs", identifier], check=False)
|
||||
stdout_str = stdout_result.stdout or ""
|
||||
stderr_str = stdout_result.stderr or ""
|
||||
|
||||
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
|
||||
return (exit_code, stdout_str, stderr_str)
|
||||
|
||||
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
|
||||
"""Execute a command inside a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param command: Command to run.
|
||||
|
||||
"""
|
||||
get_logger().debug("executing command in container", container_id=identifier)
|
||||
self._run(["exec", identifier] + command)
|
||||
|
||||
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
|
||||
"""Copy an archive to a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source archive path.
|
||||
:param destination: Destination path in container.
|
||||
|
||||
"""
|
||||
get_logger().debug("copying to container", container_id=identifier, source=str(source))
|
||||
self._run(["cp", str(source), f"{identifier}:{destination}"])
|
||||
|
||||
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
|
||||
"""Copy files from a container to a local archive.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path in container.
|
||||
:returns: Path to local archive.
|
||||
|
||||
"""
|
||||
get_logger().debug("copying from container", container_id=identifier, source=str(source))
|
||||
with NamedTemporaryFile(delete=False, delete_on_close=False, suffix=".tar") as tmp:
|
||||
self._run(["cp", f"{identifier}:{source}", tmp.name])
|
||||
return Path(tmp.name)
|
||||
|
||||
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
|
||||
"""Copy a file or directory to a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path on host.
|
||||
:param destination: Destination path in container.
|
||||
|
||||
"""
|
||||
self._run(["cp", str(source), f"{identifier}:{destination}"])
|
||||
get_logger().debug("copied to container", source=str(source), destination=destination)
|
||||
|
||||
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
|
||||
"""Copy a file or directory from a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path in container.
|
||||
:param destination: Destination path on host.
|
||||
|
||||
"""
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
self._run(["cp", f"{identifier}:{source}", str(destination)])
|
||||
get_logger().debug("copied from container", source=source, destination=str(destination))
|
||||
|
||||
def terminate_sandbox(self, identifier: str) -> None:
|
||||
"""Terminate and remove a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
|
||||
"""
|
||||
# Stop if running
|
||||
self._run(["stop", identifier], check=False)
|
||||
# Remove
|
||||
self._run(["rm", "-f", identifier], check=False)
|
||||
get_logger().debug("terminated container", container_id=identifier)
|
||||
|
||||
def remove_container(self, identifier: str, *, force: bool = False) -> None:
|
||||
"""Remove a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param force: Force removal.
|
||||
|
||||
"""
|
||||
args = ["rm"]
|
||||
if force:
|
||||
args.append("-f")
|
||||
args.append(identifier)
|
||||
self._run(args, check=False)
|
||||
get_logger().debug("removed container", container_id=identifier)
|
||||
|
||||
def stop_container(self, identifier: str, timeout: int = 10) -> None:
|
||||
"""Stop a running container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param timeout: Seconds to wait before killing.
|
||||
|
||||
"""
|
||||
self._run(["stop", "-t", str(timeout), identifier], check=False)
|
||||
get_logger().debug("stopped container", container_id=identifier)
|
||||
|
||||
def get_container_status(self, identifier: str) -> str:
|
||||
"""Get the status of a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:returns: Container status.
|
||||
|
||||
"""
|
||||
result = self._run(["inspect", "--format", "{{.State.Status}}", identifier], check=False)
|
||||
return result.stdout.strip() if result.returncode == 0 else "unknown"
|
||||
|
||||
def read_file_from_container(self, identifier: str, path: str) -> str:
|
||||
"""Read a file from inside a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param path: Path to file in container.
|
||||
:returns: File contents.
|
||||
|
||||
"""
|
||||
result = self._run(["exec", identifier, "cat", path], check=False)
|
||||
if result.returncode != 0:
|
||||
get_logger().debug("failed to read file from container", path=path)
|
||||
return ""
|
||||
return result.stdout
|
||||
|
||||
def list_containers(self, all_containers: bool = True) -> list[dict]:
|
||||
"""List containers.
|
||||
|
||||
:param all_containers: Include stopped containers.
|
||||
:returns: List of container info dicts.
|
||||
|
||||
"""
|
||||
args = ["ps", "--format", "json"]
|
||||
if all_containers:
|
||||
args.append("-a")
|
||||
|
||||
result = self._run(args)
|
||||
try:
|
||||
data = json.loads(result.stdout) if result.stdout.strip() else []
|
||||
# Handle both list and single object responses
|
||||
if isinstance(data, dict):
|
||||
data = [data]
|
||||
return [
|
||||
{
|
||||
"Id": c.get("Id", ""),
|
||||
"Names": c.get("Names", []),
|
||||
"Status": c.get("State", ""),
|
||||
"Image": c.get("Image", ""),
|
||||
}
|
||||
for c in data
|
||||
]
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Utility Methods
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def get_storage_info(self) -> dict:
|
||||
"""Get storage configuration info.
|
||||
|
||||
:returns: Dict with graphroot and runroot paths.
|
||||
|
||||
"""
|
||||
return {
|
||||
"graphroot": str(self.__graphroot),
|
||||
"runroot": str(self.__runroot),
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.base.configuration import AbstractFuzzForgeEngineConfiguration
|
||||
from fuzzforge_common.sandboxes.engines.enumeration import FuzzForgeSandboxEngines
|
||||
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine
|
||||
|
||||
|
||||
class PodmanConfiguration(AbstractFuzzForgeEngineConfiguration):
|
||||
"""TODO."""
|
||||
|
||||
#: TODO.
|
||||
kind: Literal[FuzzForgeSandboxEngines.PODMAN] = FuzzForgeSandboxEngines.PODMAN
|
||||
|
||||
#: TODO.
|
||||
socket: str
|
||||
|
||||
def into_engine(self) -> AbstractFuzzForgeSandboxEngine:
|
||||
"""TODO."""
|
||||
return Podman(socket=self.socket)
|
||||
@@ -0,0 +1,496 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tarfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path, PurePath
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from podman.errors import ImageNotFound
|
||||
|
||||
from fuzzforge_common.exceptions import FuzzForgeError
|
||||
from fuzzforge_common.sandboxes.engines.base.engine import AbstractFuzzForgeSandboxEngine, ImageInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from podman import PodmanClient
|
||||
from podman.domain.containers import Container
|
||||
from structlog.stdlib import BoundLogger
|
||||
|
||||
|
||||
def get_logger() -> BoundLogger:
|
||||
"""TODO."""
|
||||
from structlog import get_logger # noqa: PLC0415 (required by temporal)
|
||||
|
||||
return cast("BoundLogger", get_logger())
|
||||
|
||||
|
||||
class Podman(AbstractFuzzForgeSandboxEngine):
|
||||
"""TODO."""
|
||||
|
||||
#: TODO.
|
||||
__socket: str
|
||||
|
||||
def __init__(self, socket: str) -> None:
|
||||
"""Initialize an instance of the class.
|
||||
|
||||
:param socket: TODO.
|
||||
|
||||
"""
|
||||
AbstractFuzzForgeSandboxEngine.__init__(self)
|
||||
self.__socket = socket
|
||||
|
||||
def get_client(self) -> PodmanClient:
|
||||
"""TODO.
|
||||
|
||||
:returns TODO.
|
||||
|
||||
"""
|
||||
from podman import PodmanClient # noqa: PLC0415 (required by temporal)
|
||||
|
||||
return PodmanClient(base_url=self.__socket)
|
||||
|
||||
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
|
||||
"""List available container images.
|
||||
|
||||
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
|
||||
:returns: List of ImageInfo objects for available images.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
images: list[ImageInfo] = []
|
||||
|
||||
with client:
|
||||
for image in client.images.list():
|
||||
# Get all tags for this image
|
||||
tags = image.tags or []
|
||||
for tag in tags:
|
||||
# Apply filter if specified
|
||||
if filter_prefix and not tag.startswith(filter_prefix):
|
||||
continue
|
||||
|
||||
# Parse repository and tag
|
||||
if ":" in tag:
|
||||
repo, tag_name = tag.rsplit(":", 1)
|
||||
else:
|
||||
repo = tag
|
||||
tag_name = "latest"
|
||||
|
||||
images.append(
|
||||
ImageInfo(
|
||||
reference=tag,
|
||||
repository=repo,
|
||||
tag=tag_name,
|
||||
image_id=image.short_id if hasattr(image, "short_id") else image.id[:12],
|
||||
size=image.attrs.get("Size") if hasattr(image, "attrs") else None,
|
||||
)
|
||||
)
|
||||
|
||||
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
|
||||
return images
|
||||
|
||||
def register_archive(self, archive: Path, repository: str) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param archive: TODO.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
images = list(client.images.load(file_path=archive))
|
||||
if len(images) != 1:
|
||||
message: str = "expected only one image"
|
||||
raise FuzzForgeError(message)
|
||||
image = images[0]
|
||||
image.tag(repository=repository, tag="latest")
|
||||
|
||||
def spawn_sandbox(self, image: str) -> str:
|
||||
"""Spawn a sandbox based on the given image.
|
||||
|
||||
:param image: The image the sandbox should be based on.
|
||||
:returns: The sandbox identifier.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.create(image=image)
|
||||
container_identifier: str = container.id
|
||||
get_logger().debug("create podman container", container_identifier=container_identifier)
|
||||
return container_identifier
|
||||
|
||||
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: TODO.
|
||||
:param source: TODO.
|
||||
:param destination: TODO.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
get_logger().debug(
|
||||
"push archive to podman container",
|
||||
container_identifier=identifier,
|
||||
container_status=container.status,
|
||||
)
|
||||
# reading everything at once for now, even though this temporary solution is not viable with large files,
|
||||
# since the podman sdk does not currently expose a way to chunk uploads.
|
||||
# in order to fix this issue, we could directly interact with the podman rest api or make a contribution
|
||||
# to the podman sdk in order to allow the 'put_archive' method to support chunked uploads.
|
||||
data: bytes = source.read_bytes()
|
||||
container.put_archive(path=str(destination), data=data)
|
||||
|
||||
def start_sandbox(self, identifier: str) -> None:
|
||||
"""Start the sandbox matching the given identifier.
|
||||
|
||||
:param identifier: The identifier of the sandbox to start.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
get_logger().debug(
|
||||
"start podman container",
|
||||
container_identifier=identifier,
|
||||
container_status=container.status,
|
||||
)
|
||||
container.start()
|
||||
|
||||
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
|
||||
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
|
||||
|
||||
:param sandbox: The identifier of the sandbox.
|
||||
:param command: The command to run.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
get_logger().debug(
|
||||
"executing command inside podman container",
|
||||
container_identifier=identifier,
|
||||
container_status=container.status,
|
||||
)
|
||||
(status, (stdout, stderr)) = container.exec_run(cmd=command, demux=True)
|
||||
get_logger().debug(
|
||||
"command execution result",
|
||||
status=status,
|
||||
stdout_size=len(stdout) if stdout else 0,
|
||||
stderr_size=len(stderr) if stderr else 0,
|
||||
)
|
||||
|
||||
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
|
||||
"""TODO.
|
||||
|
||||
:param identifier: TODO.
|
||||
:param source: TODO.
|
||||
:returns: TODO.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
get_logger().debug(
|
||||
"pull archive from podman container",
|
||||
container_identifier=identifier,
|
||||
container_status=container.status,
|
||||
)
|
||||
with NamedTemporaryFile(delete=False, delete_on_close=False) as file:
|
||||
stream, _stat = container.get_archive(path=str(source))
|
||||
for chunk in stream:
|
||||
file.write(chunk)
|
||||
get_logger().debug(
|
||||
"created archive",
|
||||
archive=file.name,
|
||||
)
|
||||
return Path(file.name)
|
||||
|
||||
def terminate_sandbox(self, identifier: str) -> None:
|
||||
"""Terminate the sandbox matching the given identifier.
|
||||
|
||||
:param identifier: The identifier of the sandbox to terminate.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
get_logger().debug(
|
||||
"kill podman container",
|
||||
container_identifier=identifier,
|
||||
container_status=container.status,
|
||||
)
|
||||
# Only kill running containers; for created/stopped, skip to remove
|
||||
if container.status in ("running", "paused"):
|
||||
container.kill()
|
||||
get_logger().debug(
|
||||
"remove podman container",
|
||||
container_identifier=identifier,
|
||||
container_status=container.status,
|
||||
)
|
||||
container.remove()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Extended Container Operations
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def image_exists(self, image: str) -> bool:
|
||||
"""Check if a container image exists locally.
|
||||
|
||||
:param image: Full image reference (e.g., "localhost/module:latest").
|
||||
:returns: True if image exists, False otherwise.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
try:
|
||||
client.images.get(name=image)
|
||||
except ImageNotFound:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def pull_image(self, image: str, timeout: int = 300) -> None:
|
||||
"""Pull an image from a container registry.
|
||||
|
||||
:param image: Full image reference to pull.
|
||||
:param timeout: Timeout in seconds for the pull operation.
|
||||
:raises FuzzForgeError: If pull fails.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
try:
|
||||
get_logger().info("pulling image", image=image)
|
||||
client.images.pull(repository=image)
|
||||
get_logger().info("image pulled successfully", image=image)
|
||||
except Exception as exc:
|
||||
message = f"Failed to pull image '{image}': {exc}"
|
||||
raise FuzzForgeError(message) from exc
|
||||
|
||||
def tag_image(self, source: str, target: str) -> None:
|
||||
"""Tag an image with a new name.
|
||||
|
||||
:param source: Source image reference.
|
||||
:param target: Target image reference.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
image = client.images.get(name=source)
|
||||
# Parse target into repository and tag
|
||||
if ":" in target:
|
||||
repo, tag = target.rsplit(":", 1)
|
||||
else:
|
||||
repo = target
|
||||
tag = "latest"
|
||||
image.tag(repository=repo, tag=tag)
|
||||
get_logger().debug("tagged image", source=source, target=target)
|
||||
|
||||
def create_container(
|
||||
self,
|
||||
image: str,
|
||||
volumes: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Create a container from an image.
|
||||
|
||||
:param image: Image to create container from.
|
||||
:param volumes: Optional volume mappings {host_path: container_path}.
|
||||
:returns: Container identifier.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
# Build volume mounts in podman format
|
||||
mounts = []
|
||||
if volumes:
|
||||
for host_path, container_path in volumes.items():
|
||||
mounts.append({"type": "bind", "source": host_path, "target": container_path, "read_only": True})
|
||||
|
||||
container: Container = client.containers.create(image=image, mounts=mounts if mounts else None)
|
||||
container_id: str = str(container.id)
|
||||
get_logger().debug("created container", container_id=container_id, image=image)
|
||||
return container_id
|
||||
|
||||
def start_container_attached(
|
||||
self,
|
||||
identifier: str,
|
||||
timeout: int = 600,
|
||||
) -> tuple[int, str, str]:
|
||||
"""Start a container and wait for it to complete.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param timeout: Timeout in seconds for execution.
|
||||
:returns: Tuple of (exit_code, stdout, stderr).
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
get_logger().debug("starting container attached", container_id=identifier)
|
||||
|
||||
# Start the container
|
||||
container.start()
|
||||
|
||||
# Wait for completion with timeout
|
||||
result = container.wait(timeout=timeout)
|
||||
exit_code: int = result.get("StatusCode", -1) if isinstance(result, dict) else int(result)
|
||||
|
||||
# Get logs
|
||||
stdout_raw = container.logs(stdout=True, stderr=False)
|
||||
stderr_raw = container.logs(stdout=False, stderr=True)
|
||||
|
||||
# Decode if bytes
|
||||
stdout_str: str = ""
|
||||
stderr_str: str = ""
|
||||
if isinstance(stdout_raw, bytes):
|
||||
stdout_str = stdout_raw.decode("utf-8", errors="replace")
|
||||
elif isinstance(stdout_raw, str):
|
||||
stdout_str = stdout_raw
|
||||
if isinstance(stderr_raw, bytes):
|
||||
stderr_str = stderr_raw.decode("utf-8", errors="replace")
|
||||
elif isinstance(stderr_raw, str):
|
||||
stderr_str = stderr_raw
|
||||
|
||||
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
|
||||
return (exit_code, stdout_str, stderr_str)
|
||||
|
||||
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
|
||||
"""Copy a file or directory to a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path on host.
|
||||
:param destination: Destination path in container.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
|
||||
# Create tar archive in memory
|
||||
tar_buffer = BytesIO()
|
||||
with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
|
||||
tar.add(str(source), arcname=Path(source).name)
|
||||
tar_buffer.seek(0)
|
||||
|
||||
# Use put_archive to copy
|
||||
container.put_archive(path=destination, data=tar_buffer.read())
|
||||
get_logger().debug("copied to container", source=str(source), destination=destination)
|
||||
|
||||
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
|
||||
"""Copy a file or directory from a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param source: Source path in container.
|
||||
:param destination: Destination path on host.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
|
||||
# Get archive from container
|
||||
stream, _stat = container.get_archive(path=source)
|
||||
|
||||
# Write to temp file and extract
|
||||
tar_buffer = BytesIO()
|
||||
for chunk in stream:
|
||||
tar_buffer.write(chunk)
|
||||
tar_buffer.seek(0)
|
||||
|
||||
# Extract to destination
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(fileobj=tar_buffer, mode="r") as tar:
|
||||
tar.extractall(path=destination) # noqa: S202 (trusted source)
|
||||
|
||||
get_logger().debug("copied from container", source=source, destination=str(destination))
|
||||
|
||||
def remove_container(self, identifier: str, *, force: bool = False) -> None:
|
||||
"""Remove a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param force: Force removal even if running.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
if force and container.status in ("running", "paused"):
|
||||
container.kill()
|
||||
container.remove()
|
||||
get_logger().debug("removed container", container_id=identifier)
|
||||
|
||||
def start_container(self, identifier: str) -> None:
|
||||
"""Start a container without waiting for it to complete.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
container.start()
|
||||
get_logger().debug("started container (detached)", container_id=identifier)
|
||||
|
||||
def get_container_status(self, identifier: str) -> str:
|
||||
"""Get the status of a container.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:returns: Container status (e.g., "running", "exited", "created").
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
return str(container.status)
|
||||
|
||||
def stop_container(self, identifier: str, timeout: int = 10) -> None:
|
||||
"""Stop a running container gracefully.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param timeout: Seconds to wait before killing.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
if container.status == "running":
|
||||
container.stop(timeout=timeout)
|
||||
get_logger().debug("stopped container", container_id=identifier)
|
||||
|
||||
def read_file_from_container(self, identifier: str, path: str) -> str:
|
||||
"""Read a file from inside a running container using exec.
|
||||
|
||||
:param identifier: Container identifier.
|
||||
:param path: Path to file inside container.
|
||||
:returns: File contents as string.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
container: Container = client.containers.get(key=identifier)
|
||||
(status, (stdout, stderr)) = container.exec_run(cmd=["cat", path], demux=True)
|
||||
if status != 0:
|
||||
error_msg = stderr.decode("utf-8", errors="replace") if stderr else "File not found"
|
||||
get_logger().debug("failed to read file from container", path=path, error=error_msg)
|
||||
return ""
|
||||
return stdout.decode("utf-8", errors="replace") if stdout else ""
|
||||
|
||||
def list_containers(self, all_containers: bool = True) -> list[dict]:
|
||||
"""List containers.
|
||||
|
||||
:param all_containers: Include stopped containers.
|
||||
:returns: List of container info dicts.
|
||||
|
||||
"""
|
||||
client: PodmanClient = self.get_client()
|
||||
with client:
|
||||
containers = client.containers.list(all=all_containers)
|
||||
return [
|
||||
{
|
||||
"Id": str(c.id),
|
||||
"Names": [c.name] if hasattr(c, "name") else [],
|
||||
"Status": str(c.status),
|
||||
"Image": str(c.image) if hasattr(c, "image") else "",
|
||||
}
|
||||
for c in containers
|
||||
]
|
||||
19
fuzzforge-common/src/fuzzforge_common/storage/__init__.py
Normal file
19
fuzzforge-common/src/fuzzforge_common/storage/__init__.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""FuzzForge storage abstractions.
|
||||
|
||||
Storage class requires boto3. Import it explicitly:
|
||||
from fuzzforge_common.storage.s3 import Storage
|
||||
"""
|
||||
|
||||
from fuzzforge_common.storage.exceptions import (
|
||||
FuzzForgeStorageError,
|
||||
StorageConnectionError,
|
||||
StorageDownloadError,
|
||||
StorageUploadError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"FuzzForgeStorageError",
|
||||
"StorageConnectionError",
|
||||
"StorageDownloadError",
|
||||
"StorageUploadError",
|
||||
]
|
||||
@@ -0,0 +1,20 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fuzzforge_common.storage.s3 import Storage
|
||||
|
||||
|
||||
class StorageConfiguration(BaseModel):
|
||||
"""TODO."""
|
||||
|
||||
#: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO).
|
||||
endpoint: str
|
||||
|
||||
#: S3 access key ID for authentication.
|
||||
access_key: str
|
||||
|
||||
#: S3 secret access key for authentication.
|
||||
secret_key: str
|
||||
|
||||
def into_storage(self) -> Storage:
|
||||
"""TODO."""
|
||||
return Storage(endpoint=self.endpoint, access_key=self.access_key, secret_key=self.secret_key)
|
||||
108
fuzzforge-common/src/fuzzforge_common/storage/exceptions.py
Normal file
108
fuzzforge-common/src/fuzzforge_common/storage/exceptions.py
Normal file
@@ -0,0 +1,108 @@
|
||||
from fuzzforge_common.exceptions import FuzzForgeError
|
||||
|
||||
|
||||
class FuzzForgeStorageError(FuzzForgeError):
|
||||
"""Base exception for all storage-related errors.
|
||||
|
||||
Raised when storage operations (upload, download, connection) fail
|
||||
during workflow execution.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class StorageConnectionError(FuzzForgeStorageError):
|
||||
"""Failed to connect to storage service.
|
||||
|
||||
:param endpoint: The storage endpoint that failed to connect.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, endpoint: str, reason: str) -> None:
|
||||
"""Initialize storage connection error.
|
||||
|
||||
:param endpoint: The storage endpoint that failed to connect.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
FuzzForgeStorageError.__init__(
|
||||
self,
|
||||
f"Failed to connect to storage at {endpoint}: {reason}",
|
||||
)
|
||||
self.endpoint = endpoint
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class StorageUploadError(FuzzForgeStorageError):
|
||||
"""Failed to upload object to storage.
|
||||
|
||||
:param bucket: The target bucket name.
|
||||
:param object_key: The target object key.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, bucket: str, object_key: str, reason: str) -> None:
|
||||
"""Initialize storage upload error.
|
||||
|
||||
:param bucket: The target bucket name.
|
||||
:param object_key: The target object key.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
FuzzForgeStorageError.__init__(
|
||||
self,
|
||||
f"Failed to upload to {bucket}/{object_key}: {reason}",
|
||||
)
|
||||
self.bucket = bucket
|
||||
self.object_key = object_key
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class StorageDownloadError(FuzzForgeStorageError):
|
||||
"""Failed to download object from storage.
|
||||
|
||||
:param bucket: The source bucket name.
|
||||
:param object_key: The source object key.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, bucket: str, object_key: str, reason: str) -> None:
|
||||
"""Initialize storage download error.
|
||||
|
||||
:param bucket: The source bucket name.
|
||||
:param object_key: The source object key.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
FuzzForgeStorageError.__init__(
|
||||
self,
|
||||
f"Failed to download from {bucket}/{object_key}: {reason}",
|
||||
)
|
||||
self.bucket = bucket
|
||||
self.object_key = object_key
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class StorageDeletionError(FuzzForgeStorageError):
|
||||
"""Failed to delete bucket from storage.
|
||||
|
||||
:param bucket: The bucket name that failed to delete.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, bucket: str, reason: str) -> None:
|
||||
"""Initialize storage deletion error.
|
||||
|
||||
:param bucket: The bucket name that failed to delete.
|
||||
:param reason: The underlying exception message.
|
||||
|
||||
"""
|
||||
FuzzForgeStorageError.__init__(
|
||||
self,
|
||||
f"Failed to delete bucket {bucket}: {reason}",
|
||||
)
|
||||
self.bucket = bucket
|
||||
self.reason = reason
|
||||
351
fuzzforge-common/src/fuzzforge_common/storage/s3.py
Normal file
351
fuzzforge-common/src/fuzzforge_common/storage/s3.py
Normal file
@@ -0,0 +1,351 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path, PurePath
|
||||
from tarfile import TarInfo
|
||||
from tarfile import open as Archive # noqa: N812
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from fuzzforge_common.storage.exceptions import StorageDeletionError, StorageDownloadError, StorageUploadError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from botocore.client import BaseClient
|
||||
from structlog.stdlib import BoundLogger
|
||||
|
||||
|
||||
def get_logger() -> BoundLogger:
|
||||
"""Get structlog logger instance.
|
||||
|
||||
Uses deferred import pattern required by Temporal for serialization.
|
||||
|
||||
:returns: Configured structlog logger.
|
||||
|
||||
"""
|
||||
from structlog import get_logger # noqa: PLC0415 (required by temporal)
|
||||
|
||||
return cast("BoundLogger", get_logger())
|
||||
|
||||
|
||||
class Storage:
|
||||
"""S3-compatible storage backend implementation using boto3.
|
||||
|
||||
Supports MinIO, AWS S3, and other S3-compatible storage services.
|
||||
Uses error-driven approach (EAFP) to handle bucket creation and
|
||||
avoid race conditions.
|
||||
|
||||
"""
|
||||
|
||||
#: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO).
|
||||
__endpoint: str
|
||||
|
||||
#: S3 access key ID for authentication.
|
||||
__access_key: str
|
||||
|
||||
#: S3 secret access key for authentication.
|
||||
__secret_key: str
|
||||
|
||||
def __init__(self, endpoint: str, access_key: str, secret_key: str) -> None:
|
||||
"""Initialize an instance of the class.
|
||||
|
||||
:param endpoint: TODO.
|
||||
:param access_key: TODO.
|
||||
:param secret_key: TODO.
|
||||
|
||||
"""
|
||||
self.__endpoint = endpoint
|
||||
self.__access_key = access_key
|
||||
self.__secret_key = secret_key
|
||||
|
||||
def _get_client(self) -> BaseClient:
|
||||
"""Create boto3 S3 client with configured credentials.
|
||||
|
||||
Uses deferred import pattern required by Temporal for serialization.
|
||||
|
||||
:returns: Configured boto3 S3 client.
|
||||
|
||||
"""
|
||||
import boto3 # noqa: PLC0415 (required by temporal)
|
||||
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=self.__endpoint,
|
||||
aws_access_key_id=self.__access_key,
|
||||
aws_secret_access_key=self.__secret_key,
|
||||
)
|
||||
|
||||
def create_bucket(self, bucket: str) -> None:
|
||||
"""Create the S3 bucket if it does not already exist.
|
||||
|
||||
Idempotent operation - succeeds if bucket already exists and is owned by you.
|
||||
Fails if bucket exists but is owned by another account.
|
||||
|
||||
:raise ClientError: If bucket creation fails (permissions, name conflicts, etc.).
|
||||
|
||||
"""
|
||||
logger = get_logger()
|
||||
client = self._get_client()
|
||||
|
||||
logger.debug("creating_bucket", bucket=bucket)
|
||||
|
||||
try:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
logger.info("bucket_created", bucket=bucket)
|
||||
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code")
|
||||
|
||||
# Bucket already exists and we own it - this is fine
|
||||
if error_code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
logger.debug(
|
||||
"bucket_already_exists",
|
||||
bucket=bucket,
|
||||
error_code=error_code,
|
||||
)
|
||||
return
|
||||
|
||||
# Other errors are actual failures
|
||||
logger.exception(
|
||||
"bucket_creation_failed",
|
||||
bucket=bucket,
|
||||
error_code=error_code,
|
||||
)
|
||||
raise
|
||||
|
||||
def delete_bucket(self, bucket: str) -> None:
|
||||
"""Delete an S3 bucket and all its contents.
|
||||
|
||||
Idempotent operation - succeeds if bucket doesn't exist.
|
||||
Handles pagination for buckets with many objects.
|
||||
|
||||
:param bucket: The name of the bucket to delete.
|
||||
:raises StorageDeletionError: If bucket deletion fails.
|
||||
|
||||
"""
|
||||
logger = get_logger()
|
||||
client = self._get_client()
|
||||
|
||||
logger.debug("deleting_bucket", bucket=bucket)
|
||||
|
||||
try:
|
||||
# S3 requires bucket to be empty before deletion
|
||||
# Delete all objects first with pagination support
|
||||
continuation_token = None
|
||||
|
||||
while True:
|
||||
# List objects (up to 1000 per request)
|
||||
list_params = {"Bucket": bucket}
|
||||
if continuation_token:
|
||||
list_params["ContinuationToken"] = continuation_token
|
||||
|
||||
response = client.list_objects_v2(**list_params)
|
||||
|
||||
# Delete objects if any exist (max 1000 per delete_objects call)
|
||||
if "Contents" in response:
|
||||
objects = [{"Key": obj["Key"]} for obj in response["Contents"]]
|
||||
client.delete_objects(Bucket=bucket, Delete={"Objects": objects})
|
||||
logger.debug("deleted_objects", bucket=bucket, count=len(objects))
|
||||
|
||||
# Check if more objects exist
|
||||
if not response.get("IsTruncated", False):
|
||||
break
|
||||
|
||||
continuation_token = response.get("NextContinuationToken")
|
||||
|
||||
# Now delete the empty bucket
|
||||
client.delete_bucket(Bucket=bucket)
|
||||
logger.info("bucket_deleted", bucket=bucket)
|
||||
|
||||
except ClientError as error:
|
||||
error_code = error.response.get("Error", {}).get("Code")
|
||||
|
||||
# Idempotent - bucket already doesn't exist
|
||||
if error_code == "NoSuchBucket":
|
||||
logger.debug("bucket_does_not_exist", bucket=bucket)
|
||||
return
|
||||
|
||||
# Other errors are actual failures
|
||||
logger.exception(
|
||||
"bucket_deletion_failed",
|
||||
bucket=bucket,
|
||||
error_code=error_code,
|
||||
)
|
||||
raise StorageDeletionError(bucket=bucket, reason=str(error)) from error
|
||||
|
||||
def upload_file(
|
||||
self,
|
||||
bucket: str,
|
||||
file: Path,
|
||||
key: str,
|
||||
) -> None:
|
||||
"""Upload archive file to S3 storage at specified object key.
|
||||
|
||||
Assumes bucket exists. Fails gracefully if bucket or other resources missing.
|
||||
|
||||
:param bucket: TODO.
|
||||
:param file: Local path to the archive file to upload.
|
||||
:param key: Object key (path) in S3 where file should be uploaded.
|
||||
:raise StorageUploadError: If upload operation fails.
|
||||
|
||||
"""
|
||||
from boto3.exceptions import S3UploadFailedError # noqa: PLC0415 (required by 'temporal' at runtime)
|
||||
|
||||
logger = get_logger()
|
||||
client = self._get_client()
|
||||
|
||||
logger.debug(
|
||||
"uploading_archive_to_storage",
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
archive_path=str(file),
|
||||
)
|
||||
|
||||
try:
|
||||
client.upload_file(
|
||||
Filename=str(file),
|
||||
Bucket=bucket,
|
||||
Key=key,
|
||||
)
|
||||
logger.info(
|
||||
"archive_uploaded_successfully",
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
)
|
||||
|
||||
except S3UploadFailedError as e:
|
||||
# Check if this is a NoSuchBucket error - create bucket and retry
|
||||
if "NoSuchBucket" in str(e):
|
||||
logger.info(
|
||||
"bucket_does_not_exist_creating",
|
||||
bucket=bucket,
|
||||
)
|
||||
self.create_bucket(bucket=bucket)
|
||||
# Retry upload after creating bucket
|
||||
try:
|
||||
client.upload_file(
|
||||
Filename=str(file),
|
||||
Bucket=bucket,
|
||||
Key=key,
|
||||
)
|
||||
logger.info(
|
||||
"archive_uploaded_successfully_after_bucket_creation",
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
)
|
||||
except S3UploadFailedError as retry_error:
|
||||
logger.exception(
|
||||
"upload_failed_after_bucket_creation",
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
)
|
||||
raise StorageUploadError(
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
reason=str(retry_error),
|
||||
) from retry_error
|
||||
else:
|
||||
logger.exception(
|
||||
"upload_failed",
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
)
|
||||
raise StorageUploadError(
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
reason=str(e),
|
||||
) from e
|
||||
|
||||
def download_file(self, bucket: str, key: PurePath) -> Path:
|
||||
"""Download a single file from S3 storage.
|
||||
|
||||
Downloads the file to a temporary location and returns the path.
|
||||
|
||||
:param bucket: S3 bucket name.
|
||||
:param key: Object key (path) in S3 to download.
|
||||
:returns: Path to the downloaded file.
|
||||
:raise StorageDownloadError: If download operation fails.
|
||||
|
||||
"""
|
||||
logger = get_logger()
|
||||
client = self._get_client()
|
||||
|
||||
logger.debug(
|
||||
"downloading_file_from_storage",
|
||||
bucket=bucket,
|
||||
object_key=str(key),
|
||||
)
|
||||
|
||||
try:
|
||||
# Create temporary file for download
|
||||
with NamedTemporaryFile(delete=False, suffix=".tar.gz") as temp_file:
|
||||
temp_path = Path(temp_file.name)
|
||||
|
||||
# Download object to temp file
|
||||
client.download_file(
|
||||
Bucket=bucket,
|
||||
Key=str(key),
|
||||
Filename=str(temp_path),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"file_downloaded_successfully",
|
||||
bucket=bucket,
|
||||
object_key=str(key),
|
||||
local_path=str(temp_path),
|
||||
)
|
||||
|
||||
return temp_path
|
||||
|
||||
except ClientError as error:
|
||||
error_code = error.response.get("Error", {}).get("Code")
|
||||
logger.exception(
|
||||
"download_failed",
|
||||
bucket=bucket,
|
||||
object_key=str(key),
|
||||
error_code=error_code,
|
||||
)
|
||||
raise StorageDownloadError(
|
||||
bucket=bucket,
|
||||
object_key=str(key),
|
||||
reason=f"{error_code}: {error!s}",
|
||||
) from error
|
||||
|
||||
def download_directory(self, bucket: str, directory: PurePath) -> Path:
|
||||
"""TODO.
|
||||
|
||||
:param bucket: TODO.
|
||||
:param directory: TODO.
|
||||
:returns: TODO.
|
||||
|
||||
"""
|
||||
with NamedTemporaryFile(delete=False) as file:
|
||||
path: Path = Path(file.name)
|
||||
# end-with
|
||||
client: Any = self._get_client()
|
||||
with Archive(name=str(path), mode="w:gz") as archive:
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
pages = paginator.paginate(Bucket=bucket, Prefix=str(directory))
|
||||
except ClientError as exception:
|
||||
raise StorageDownloadError(
|
||||
bucket=bucket,
|
||||
object_key=str(directory),
|
||||
reason=exception.response["Error"]["Code"],
|
||||
) from exception
|
||||
for page in pages:
|
||||
for entry in page.get("Contents", []):
|
||||
key: str = entry["Key"]
|
||||
try:
|
||||
response: dict[str, Any] = client.get_object(Bucket=bucket, Key=key)
|
||||
except ClientError as exception:
|
||||
raise StorageDownloadError(
|
||||
bucket=bucket,
|
||||
object_key=key,
|
||||
reason=exception.response["Error"]["Code"],
|
||||
) from exception
|
||||
archive.addfile(TarInfo(name=key), fileobj=response["Body"])
|
||||
# end-for
|
||||
# end-for
|
||||
# end-with
|
||||
return path
|
||||
8
fuzzforge-common/src/fuzzforge_common/temporal/queues.py
Normal file
8
fuzzforge-common/src/fuzzforge_common/temporal/queues.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class TemporalQueues(StrEnum):
|
||||
"""Enumeration of available `Temporal Task Queues`."""
|
||||
|
||||
#: The default task queue.
|
||||
DEFAULT = "default-task-queue"
|
||||
@@ -0,0 +1,46 @@
|
||||
from enum import StrEnum
|
||||
from typing import Literal
|
||||
|
||||
from fuzzforge_types import FuzzForgeWorkflowIdentifier # noqa: TC002 (required by 'pydantic' at runtime)
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Base(BaseModel):
|
||||
"""TODO."""
|
||||
|
||||
|
||||
class FuzzForgeWorkflowSteps(StrEnum):
|
||||
"""Workflow step types."""
|
||||
|
||||
#: Execute a FuzzForge module
|
||||
RUN_FUZZFORGE_MODULE = "run-fuzzforge-module"
|
||||
|
||||
|
||||
class FuzzForgeWorkflowStep(Base):
|
||||
"""TODO."""
|
||||
|
||||
#: The type of the workflow's step.
|
||||
kind: FuzzForgeWorkflowSteps
|
||||
|
||||
|
||||
class RunFuzzForgeModule(FuzzForgeWorkflowStep):
|
||||
"""Execute a FuzzForge module."""
|
||||
|
||||
kind: Literal[FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE] = FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE
|
||||
#: The name of the module.
|
||||
module: str
|
||||
#: The container of the module.
|
||||
container: str
|
||||
|
||||
|
||||
class FuzzForgeWorkflowDefinition(Base):
|
||||
"""The definition of a FuzzForge workflow."""
|
||||
|
||||
#: The author of the workflow.
|
||||
author: str
|
||||
#: The identifier of the workflow.
|
||||
identifier: FuzzForgeWorkflowIdentifier
|
||||
#: The name of the workflow.
|
||||
name: str
|
||||
#: The collection of steps that compose the workflow.
|
||||
steps: list[RunFuzzForgeModule]
|
||||
@@ -0,0 +1,24 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.docker.configuration import (
|
||||
DockerConfiguration, # noqa: TC001 (required by pydantic at runtime)
|
||||
)
|
||||
from fuzzforge_common.sandboxes.engines.podman.configuration import (
|
||||
PodmanConfiguration, # noqa: TC001 (required by pydantic at runtime)
|
||||
)
|
||||
from fuzzforge_common.storage.configuration import StorageConfiguration # noqa: TC001 (required by pydantic at runtime)
|
||||
|
||||
|
||||
class TemporalWorkflowParameters(BaseModel):
|
||||
"""Base parameters for Temporal workflows.
|
||||
|
||||
Provides common configuration shared across all workflow types,
|
||||
including sandbox engine and storage backend instances.
|
||||
|
||||
"""
|
||||
|
||||
#: Sandbox engine for container operations (Docker or Podman).
|
||||
engine_configuration: PodmanConfiguration | DockerConfiguration
|
||||
|
||||
#: Storage backend for uploading/downloading execution artifacts.
|
||||
storage_configuration: StorageConfiguration
|
||||
108
fuzzforge-common/src/fuzzforge_common/workflows/bridge_utils.py
Normal file
108
fuzzforge-common/src/fuzzforge_common/workflows/bridge_utils.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""Helper utilities for working with bridge transformations."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_transform_from_file(file_path: str | Path) -> str:
|
||||
"""Load bridge transformation code from a Python file.
|
||||
|
||||
This reads the transformation function from a .py file and extracts
|
||||
the code as a string suitable for the bridge module.
|
||||
|
||||
Args:
|
||||
file_path: Path to Python file containing transform() function
|
||||
|
||||
Returns:
|
||||
Python code as a string
|
||||
|
||||
Example:
|
||||
>>> code = load_transform_from_file("transformations/add_line_numbers.py")
|
||||
>>> # code contains the transform() function as a string
|
||||
|
||||
"""
|
||||
path = Path(file_path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Transformation file not found: {file_path}")
|
||||
|
||||
if path.suffix != ".py":
|
||||
raise ValueError(f"Transformation file must be .py file, got: {path.suffix}")
|
||||
|
||||
# Read the entire file
|
||||
code = path.read_text()
|
||||
|
||||
return code
|
||||
|
||||
|
||||
def create_bridge_input(
|
||||
transform_file: str | Path,
|
||||
input_filename: str | None = None,
|
||||
output_filename: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create bridge module input configuration from a transformation file.
|
||||
|
||||
Args:
|
||||
transform_file: Path to Python file with transform() function
|
||||
input_filename: Optional specific input file to transform
|
||||
output_filename: Optional specific output filename
|
||||
|
||||
Returns:
|
||||
Dictionary suitable for bridge module's input.json
|
||||
|
||||
Example:
|
||||
>>> config = create_bridge_input("transformations/add_line_numbers.py")
|
||||
>>> import json
|
||||
>>> json.dump(config, open("input.json", "w"))
|
||||
|
||||
"""
|
||||
code = load_transform_from_file(transform_file)
|
||||
|
||||
return {
|
||||
"code": code,
|
||||
"input_filename": input_filename,
|
||||
"output_filename": output_filename,
|
||||
}
|
||||
|
||||
|
||||
def validate_transform_function(file_path: str | Path) -> bool:
|
||||
"""Validate that a Python file contains a valid transform() function.
|
||||
|
||||
Args:
|
||||
file_path: Path to Python file to validate
|
||||
|
||||
Returns:
|
||||
True if valid, raises exception otherwise
|
||||
|
||||
Raises:
|
||||
ValueError: If transform() function is not found or invalid
|
||||
|
||||
"""
|
||||
code = load_transform_from_file(file_path)
|
||||
|
||||
# Check if transform function is defined
|
||||
if "def transform(" not in code:
|
||||
raise ValueError(
|
||||
f"File {file_path} must contain a 'def transform(data)' function"
|
||||
)
|
||||
|
||||
# Try to compile the code
|
||||
try:
|
||||
compile(code, str(file_path), "exec")
|
||||
except SyntaxError as e:
|
||||
raise ValueError(f"Syntax error in {file_path}: {e}") from e
|
||||
|
||||
# Try to execute and verify transform exists
|
||||
namespace: dict[str, Any] = {"__builtins__": __builtins__}
|
||||
try:
|
||||
exec(code, namespace)
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to execute {file_path}: {e}") from e
|
||||
|
||||
if "transform" not in namespace:
|
||||
raise ValueError(f"No 'transform' function found in {file_path}")
|
||||
|
||||
if not callable(namespace["transform"]):
|
||||
raise ValueError(f"'transform' in {file_path} is not callable")
|
||||
|
||||
return True
|
||||
27
fuzzforge-common/src/fuzzforge_common/workflows/default.py
Normal file
27
fuzzforge-common/src/fuzzforge_common/workflows/default.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from fuzzforge_types import (
|
||||
FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime)
|
||||
FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime)
|
||||
)
|
||||
|
||||
from fuzzforge_common.workflows.base.definitions import (
|
||||
FuzzForgeWorkflowDefinition, # noqa: TC001 (required by pydantic at runtime)
|
||||
)
|
||||
from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters
|
||||
|
||||
|
||||
class ExecuteFuzzForgeWorkflowParameters(TemporalWorkflowParameters):
|
||||
"""Parameters for the default FuzzForge workflow orchestration.
|
||||
|
||||
Contains workflow definition and execution tracking identifiers
|
||||
for coordinating multi-module workflows.
|
||||
|
||||
"""
|
||||
|
||||
#: UUID7 identifier of this specific workflow execution.
|
||||
execution_identifier: FuzzForgeExecutionIdentifier
|
||||
|
||||
#: UUID7 identifier of the project this execution belongs to.
|
||||
project_identifier: FuzzForgeProjectIdentifier
|
||||
|
||||
#: The definition of the FuzzForge workflow to run.
|
||||
workflow_definition: FuzzForgeWorkflowDefinition
|
||||
80
fuzzforge-common/src/fuzzforge_common/workflows/modules.py
Normal file
80
fuzzforge-common/src/fuzzforge_common/workflows/modules.py
Normal file
@@ -0,0 +1,80 @@
|
||||
from typing import Any, Literal
|
||||
|
||||
from fuzzforge_types import (
|
||||
FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime)
|
||||
FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime)
|
||||
)
|
||||
|
||||
from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters
|
||||
|
||||
|
||||
class ExecuteFuzzForgeModuleParameters(TemporalWorkflowParameters):
|
||||
"""Parameters for executing a single FuzzForge module workflow.
|
||||
|
||||
Contains module execution configuration including container image,
|
||||
project context, and execution tracking identifiers.
|
||||
|
||||
Supports workflow chaining where modules can be executed in sequence,
|
||||
with each module's output becoming the next module's input.
|
||||
|
||||
"""
|
||||
|
||||
#: The identifier of this module execution.
|
||||
execution_identifier: FuzzForgeExecutionIdentifier
|
||||
|
||||
#: The identifier/name of the module to execute.
|
||||
#: FIXME: Currently accepts both UUID (for registry lookups) and container names (e.g., "text-generator:0.0.1").
|
||||
#: This should be split into module_identifier (UUID) and container_image (string) in the future.
|
||||
module_identifier: str
|
||||
|
||||
#: The identifier of the project this module execution belongs to.
|
||||
project_identifier: FuzzForgeProjectIdentifier
|
||||
|
||||
#: Optional configuration dictionary for the module.
|
||||
#: Will be written to /data/input/config.json in the sandbox.
|
||||
module_configuration: dict[str, Any] | None = None
|
||||
|
||||
# Workflow chaining fields
|
||||
|
||||
#: The identifier of the parent workflow execution (if part of a multi-module workflow).
|
||||
#: For standalone module executions, this equals execution_identifier.
|
||||
workflow_execution_identifier: FuzzForgeExecutionIdentifier | None = None
|
||||
|
||||
#: Position of this module in the workflow (0-based).
|
||||
#: 0 = first module (reads from project assets)
|
||||
#: N > 0 = subsequent module (reads from previous module's output)
|
||||
step_index: int = 0
|
||||
|
||||
#: Execution identifier of the previous module in the workflow chain.
|
||||
#: None for first module (step_index=0).
|
||||
#: Used to locate previous module's output in storage.
|
||||
previous_step_execution_identifier: FuzzForgeExecutionIdentifier | None = None
|
||||
|
||||
|
||||
class WorkflowStep(TemporalWorkflowParameters):
|
||||
"""A step in a workflow - a module execution.
|
||||
|
||||
Steps are executed sequentially in a workflow. Each step runs a containerized module.
|
||||
|
||||
Examples:
|
||||
# Module step
|
||||
WorkflowStep(
|
||||
step_index=0,
|
||||
step_type="module",
|
||||
module_identifier="text-generator:0.0.1"
|
||||
)
|
||||
|
||||
"""
|
||||
|
||||
#: Position of this step in the workflow (0-based)
|
||||
step_index: int
|
||||
|
||||
#: Type of step: "module" (bridges are also modules now)
|
||||
step_type: Literal["module"]
|
||||
|
||||
#: Module identifier (container image name like "text-generator:0.0.1")
|
||||
#: Required if step_type="module"
|
||||
module_identifier: str | None = None
|
||||
|
||||
#: Optional module configuration
|
||||
module_configuration: dict[str, Any] | None = None
|
||||
0
fuzzforge-common/tests/.gitkeep
Normal file
0
fuzzforge-common/tests/.gitkeep
Normal file
0
fuzzforge-common/tests/__init__.py
Normal file
0
fuzzforge-common/tests/__init__.py
Normal file
1
fuzzforge-common/tests/conftest.py
Normal file
1
fuzzforge-common/tests/conftest.py
Normal file
@@ -0,0 +1 @@
|
||||
pytest_plugins = ["fuzzforge_tests.fixtures"]
|
||||
0
fuzzforge-common/tests/unit/__init__.py
Normal file
0
fuzzforge-common/tests/unit/__init__.py
Normal file
0
fuzzforge-common/tests/unit/engines/__init__.py
Normal file
0
fuzzforge-common/tests/unit/engines/__init__.py
Normal file
9
fuzzforge-common/tests/unit/engines/conftest.py
Normal file
9
fuzzforge-common/tests/unit/engines/conftest.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import pytest
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def podman_engine(podman_socket: str) -> Podman:
|
||||
"""TODO."""
|
||||
return Podman(socket=podman_socket)
|
||||
21
fuzzforge-common/tests/unit/engines/test_podman.py
Normal file
21
fuzzforge-common/tests/unit/engines/test_podman.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from typing import TYPE_CHECKING
|
||||
from uuid import uuid4
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from podman import PodmanClient
|
||||
|
||||
from fuzzforge_common.sandboxes.engines.podman.engine import Podman
|
||||
|
||||
|
||||
def test_can_register_oci(
|
||||
path_to_oci: Path,
|
||||
podman_engine: Podman,
|
||||
podman_client: PodmanClient,
|
||||
) -> None:
|
||||
"""TODO."""
|
||||
repository: str = str(uuid4())
|
||||
podman_engine.register_archive(archive=path_to_oci, repository=repository)
|
||||
assert podman_client.images.exists(key=repository)
|
||||
podman_client.images.get(name=repository).remove()
|
||||
0
fuzzforge-common/tests/unit/storage/__init__.py
Normal file
0
fuzzforge-common/tests/unit/storage/__init__.py
Normal file
42
fuzzforge-common/tests/unit/storage/test_storage.py
Normal file
42
fuzzforge-common/tests/unit/storage/test_storage.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fuzzforge_common.storage.configuration import StorageConfiguration
|
||||
|
||||
|
||||
def test_download_directory(
|
||||
storage_configuration: StorageConfiguration,
|
||||
boto3_client: Any,
|
||||
random_bucket: str,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""TODO."""
|
||||
bucket = random_bucket
|
||||
storage = storage_configuration.into_storage()
|
||||
|
||||
d1 = tmp_path.joinpath("d1")
|
||||
f1 = d1.joinpath("f1")
|
||||
d2 = tmp_path.joinpath("d2")
|
||||
f2 = d2.joinpath("f2")
|
||||
d3 = d2.joinpath("d3")
|
||||
f3 = d3.joinpath("d3")
|
||||
|
||||
d1.mkdir()
|
||||
d2.mkdir()
|
||||
d3.mkdir()
|
||||
f1.touch()
|
||||
f2.touch()
|
||||
f3.touch()
|
||||
|
||||
for path in [f1, f2, f3]:
|
||||
key: Path = Path("assets", path.relative_to(other=tmp_path))
|
||||
boto3_client.upload_file(
|
||||
Bucket=bucket,
|
||||
Filename=str(path),
|
||||
Key=str(key),
|
||||
)
|
||||
|
||||
path = storage.download_directory(bucket=bucket, directory="assets")
|
||||
|
||||
assert path.is_file()
|
||||
Reference in New Issue
Block a user