CalvinBackup/fuzzforge_ai
1
0
Fork 0
mirror of https://github.com/FuzzingLabs/fuzzforge_ai.git synced 2026-07-07 14:57:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: remove dead code from OSS (fuzzforge-types and unused fuzzforge-common modules)

Browse Source
This commit is contained in:
AFredefon
2026-02-18 01:38:35 +01:00
parent be55bd3426
commit 829e8b994b
41 changed files with 31 additions and 1277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fuzzforge-common/src/fuzzforge_common/__init__.py
-16
View File
@@ -2,7 +2,6 @@
This package provides:
- Sandbox engine abstractions (Podman, Docker)
- Storage abstractions (S3) - requires 'storage' extra
- Common exceptions
Example usage:
@@ -12,9 +11,6 @@ Example usage:
Podman,
PodmanConfiguration,
)
# For storage (requires boto3):
from fuzzforge_common.storage import Storage
"""
from fuzzforge_common.exceptions import FuzzForgeError
@@ -29,14 +25,6 @@ from fuzzforge_common.sandboxes import (
PodmanConfiguration,
)
# Storage exceptions are always available (no boto3 required)
from fuzzforge_common.storage.exceptions import (
FuzzForgeStorageError,
StorageConnectionError,
StorageDownloadError,
StorageUploadError,
)
__all__ = [
"AbstractFuzzForgeEngineConfiguration",
"AbstractFuzzForgeSandboxEngine",
@@ -44,11 +32,7 @@ __all__ = [
"DockerConfiguration",
"FuzzForgeError",
"FuzzForgeSandboxEngines",
"FuzzForgeStorageError",
"ImageInfo",
"Podman",
"PodmanConfiguration",
"StorageConnectionError",
"StorageDownloadError",
"StorageUploadError",
]
fuzzforge-common/src/fuzzforge_common/storage/__init__.py
-19
View File
@@ -1,19 +0,0 @@
"""FuzzForge storage abstractions.
Storage class requires boto3. Import it explicitly:
from fuzzforge_common.storage.s3 import Storage
"""
from fuzzforge_common.storage.exceptions import (
FuzzForgeStorageError,
StorageConnectionError,
StorageDownloadError,
StorageUploadError,
)
__all__ = [
"FuzzForgeStorageError",
"StorageConnectionError",
"StorageDownloadError",
"StorageUploadError",
]
fuzzforge-common/src/fuzzforge_common/storage/configuration.py
-20
View File
@@ -1,20 +0,0 @@
from pydantic import BaseModel
from fuzzforge_common.storage.s3 import Storage
class StorageConfiguration(BaseModel):
"""TODO."""
#: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO).
endpoint: str
#: S3 access key ID for authentication.
access_key: str
#: S3 secret access key for authentication.
secret_key: str
def into_storage(self) -> Storage:
"""TODO."""
return Storage(endpoint=self.endpoint, access_key=self.access_key, secret_key=self.secret_key)
fuzzforge-common/src/fuzzforge_common/storage/exceptions.py
-108
View File
@@ -1,108 +0,0 @@
from fuzzforge_common.exceptions import FuzzForgeError
class FuzzForgeStorageError(FuzzForgeError):
"""Base exception for all storage-related errors.
Raised when storage operations (upload, download, connection) fail
during workflow execution.
"""
class StorageConnectionError(FuzzForgeStorageError):
"""Failed to connect to storage service.
:param endpoint: The storage endpoint that failed to connect.
:param reason: The underlying exception message.
"""
def __init__(self, endpoint: str, reason: str) -> None:
"""Initialize storage connection error.
:param endpoint: The storage endpoint that failed to connect.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to connect to storage at {endpoint}: {reason}",
)
self.endpoint = endpoint
self.reason = reason
class StorageUploadError(FuzzForgeStorageError):
"""Failed to upload object to storage.
:param bucket: The target bucket name.
:param object_key: The target object key.
:param reason: The underlying exception message.
"""
def __init__(self, bucket: str, object_key: str, reason: str) -> None:
"""Initialize storage upload error.
:param bucket: The target bucket name.
:param object_key: The target object key.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to upload to {bucket}/{object_key}: {reason}",
)
self.bucket = bucket
self.object_key = object_key
self.reason = reason
class StorageDownloadError(FuzzForgeStorageError):
"""Failed to download object from storage.
:param bucket: The source bucket name.
:param object_key: The source object key.
:param reason: The underlying exception message.
"""
def __init__(self, bucket: str, object_key: str, reason: str) -> None:
"""Initialize storage download error.
:param bucket: The source bucket name.
:param object_key: The source object key.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to download from {bucket}/{object_key}: {reason}",
)
self.bucket = bucket
self.object_key = object_key
self.reason = reason
class StorageDeletionError(FuzzForgeStorageError):
"""Failed to delete bucket from storage.
:param bucket: The bucket name that failed to delete.
:param reason: The underlying exception message.
"""
def __init__(self, bucket: str, reason: str) -> None:
"""Initialize storage deletion error.
:param bucket: The bucket name that failed to delete.
:param reason: The underlying exception message.
"""
FuzzForgeStorageError.__init__(
self,
f"Failed to delete bucket {bucket}: {reason}",
)
self.bucket = bucket
self.reason = reason
fuzzforge-common/src/fuzzforge_common/storage/s3.py
-351
View File
@@ -1,351 +0,0 @@
from __future__ import annotations
from pathlib import Path, PurePath
from tarfile import TarInfo
from tarfile import open as Archive # noqa: N812
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any, cast
from botocore.exceptions import ClientError
from fuzzforge_common.storage.exceptions import StorageDeletionError, StorageDownloadError, StorageUploadError
if TYPE_CHECKING:
from botocore.client import BaseClient
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structlog logger instance.
Uses deferred import pattern required by Temporal for serialization.
:returns: Configured structlog logger.
"""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
class Storage:
"""S3-compatible storage backend implementation using boto3.
Supports MinIO, AWS S3, and other S3-compatible storage services.
Uses error-driven approach (EAFP) to handle bucket creation and
avoid race conditions.
"""
#: S3 endpoint URL (e.g., "http://localhost:9000" for MinIO).
__endpoint: str
#: S3 access key ID for authentication.
__access_key: str
#: S3 secret access key for authentication.
__secret_key: str
def __init__(self, endpoint: str, access_key: str, secret_key: str) -> None:
"""Initialize an instance of the class.
:param endpoint: TODO.
:param access_key: TODO.
:param secret_key: TODO.
"""
self.__endpoint = endpoint
self.__access_key = access_key
self.__secret_key = secret_key
def _get_client(self) -> BaseClient:
"""Create boto3 S3 client with configured credentials.
Uses deferred import pattern required by Temporal for serialization.
:returns: Configured boto3 S3 client.
"""
import boto3 # noqa: PLC0415 (required by temporal)
return boto3.client(
"s3",
endpoint_url=self.__endpoint,
aws_access_key_id=self.__access_key,
aws_secret_access_key=self.__secret_key,
)
def create_bucket(self, bucket: str) -> None:
"""Create the S3 bucket if it does not already exist.
Idempotent operation - succeeds if bucket already exists and is owned by you.
Fails if bucket exists but is owned by another account.
:raise ClientError: If bucket creation fails (permissions, name conflicts, etc.).
"""
logger = get_logger()
client = self._get_client()
logger.debug("creating_bucket", bucket=bucket)
try:
client.create_bucket(Bucket=bucket)
logger.info("bucket_created", bucket=bucket)
except ClientError as e:
error_code = e.response.get("Error", {}).get("Code")
# Bucket already exists and we own it - this is fine
if error_code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
logger.debug(
"bucket_already_exists",
bucket=bucket,
error_code=error_code,
)
return
# Other errors are actual failures
logger.exception(
"bucket_creation_failed",
bucket=bucket,
error_code=error_code,
)
raise
def delete_bucket(self, bucket: str) -> None:
"""Delete an S3 bucket and all its contents.
Idempotent operation - succeeds if bucket doesn't exist.
Handles pagination for buckets with many objects.
:param bucket: The name of the bucket to delete.
:raises StorageDeletionError: If bucket deletion fails.
"""
logger = get_logger()
client = self._get_client()
logger.debug("deleting_bucket", bucket=bucket)
try:
# S3 requires bucket to be empty before deletion
# Delete all objects first with pagination support
continuation_token = None
while True:
# List objects (up to 1000 per request)
list_params = {"Bucket": bucket}
if continuation_token:
list_params["ContinuationToken"] = continuation_token
response = client.list_objects_v2(**list_params)
# Delete objects if any exist (max 1000 per delete_objects call)
if "Contents" in response:
objects = [{"Key": obj["Key"]} for obj in response["Contents"]]
client.delete_objects(Bucket=bucket, Delete={"Objects": objects})
logger.debug("deleted_objects", bucket=bucket, count=len(objects))
# Check if more objects exist
if not response.get("IsTruncated", False):
break
continuation_token = response.get("NextContinuationToken")
# Now delete the empty bucket
client.delete_bucket(Bucket=bucket)
logger.info("bucket_deleted", bucket=bucket)
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
# Idempotent - bucket already doesn't exist
if error_code == "NoSuchBucket":
logger.debug("bucket_does_not_exist", bucket=bucket)
return
# Other errors are actual failures
logger.exception(
"bucket_deletion_failed",
bucket=bucket,
error_code=error_code,
)
raise StorageDeletionError(bucket=bucket, reason=str(error)) from error
def upload_file(
self,
bucket: str,
file: Path,
key: str,
) -> None:
"""Upload archive file to S3 storage at specified object key.
Assumes bucket exists. Fails gracefully if bucket or other resources missing.
:param bucket: TODO.
:param file: Local path to the archive file to upload.
:param key: Object key (path) in S3 where file should be uploaded.
:raise StorageUploadError: If upload operation fails.
"""
from boto3.exceptions import S3UploadFailedError # noqa: PLC0415 (required by 'temporal' at runtime)
logger = get_logger()
client = self._get_client()
logger.debug(
"uploading_archive_to_storage",
bucket=bucket,
object_key=key,
archive_path=str(file),
)
try:
client.upload_file(
Filename=str(file),
Bucket=bucket,
Key=key,
)
logger.info(
"archive_uploaded_successfully",
bucket=bucket,
object_key=key,
)
except S3UploadFailedError as e:
# Check if this is a NoSuchBucket error - create bucket and retry
if "NoSuchBucket" in str(e):
logger.info(
"bucket_does_not_exist_creating",
bucket=bucket,
)
self.create_bucket(bucket=bucket)
# Retry upload after creating bucket
try:
client.upload_file(
Filename=str(file),
Bucket=bucket,
Key=key,
)
logger.info(
"archive_uploaded_successfully_after_bucket_creation",
bucket=bucket,
object_key=key,
)
except S3UploadFailedError as retry_error:
logger.exception(
"upload_failed_after_bucket_creation",
bucket=bucket,
object_key=key,
)
raise StorageUploadError(
bucket=bucket,
object_key=key,
reason=str(retry_error),
) from retry_error
else:
logger.exception(
"upload_failed",
bucket=bucket,
object_key=key,
)
raise StorageUploadError(
bucket=bucket,
object_key=key,
reason=str(e),
) from e
def download_file(self, bucket: str, key: PurePath) -> Path:
"""Download a single file from S3 storage.
Downloads the file to a temporary location and returns the path.
:param bucket: S3 bucket name.
:param key: Object key (path) in S3 to download.
:returns: Path to the downloaded file.
:raise StorageDownloadError: If download operation fails.
"""
logger = get_logger()
client = self._get_client()
logger.debug(
"downloading_file_from_storage",
bucket=bucket,
object_key=str(key),
)
try:
# Create temporary file for download
with NamedTemporaryFile(delete=False, suffix=".tar.gz") as temp_file:
temp_path = Path(temp_file.name)
# Download object to temp file
client.download_file(
Bucket=bucket,
Key=str(key),
Filename=str(temp_path),
)
logger.info(
"file_downloaded_successfully",
bucket=bucket,
object_key=str(key),
local_path=str(temp_path),
)
return temp_path
except ClientError as error:
error_code = error.response.get("Error", {}).get("Code")
logger.exception(
"download_failed",
bucket=bucket,
object_key=str(key),
error_code=error_code,
)
raise StorageDownloadError(
bucket=bucket,
object_key=str(key),
reason=f"{error_code}: {error!s}",
) from error
def download_directory(self, bucket: str, directory: PurePath) -> Path:
"""TODO.
:param bucket: TODO.
:param directory: TODO.
:returns: TODO.
"""
with NamedTemporaryFile(delete=False) as file:
path: Path = Path(file.name)
# end-with
client: Any = self._get_client()
with Archive(name=str(path), mode="w:gz") as archive:
paginator = client.get_paginator("list_objects_v2")
try:
pages = paginator.paginate(Bucket=bucket, Prefix=str(directory))
except ClientError as exception:
raise StorageDownloadError(
bucket=bucket,
object_key=str(directory),
reason=exception.response["Error"]["Code"],
) from exception
for page in pages:
for entry in page.get("Contents", []):
key: str = entry["Key"]
try:
response: dict[str, Any] = client.get_object(Bucket=bucket, Key=key)
except ClientError as exception:
raise StorageDownloadError(
bucket=bucket,
object_key=key,
reason=exception.response["Error"]["Code"],
) from exception
archive.addfile(TarInfo(name=key), fileobj=response["Body"])
# end-for
# end-for
# end-with
return path
fuzzforge-common/src/fuzzforge_common/temporal/__init__.py
View File
fuzzforge-common/src/fuzzforge_common/temporal/queues.py
-8
View File
@@ -1,8 +0,0 @@
from enum import StrEnum
class TemporalQueues(StrEnum):
"""Enumeration of available `Temporal Task Queues`."""
#: The default task queue.
DEFAULT = "default-task-queue"
fuzzforge-common/src/fuzzforge_common/workflows/__init__.py
View File
fuzzforge-common/src/fuzzforge_common/workflows/base/__init__.py
View File
fuzzforge-common/src/fuzzforge_common/workflows/base/definitions.py
-46
View File
@@ -1,46 +0,0 @@
from enum import StrEnum
from typing import Literal
from fuzzforge_types import FuzzForgeWorkflowIdentifier # noqa: TC002 (required by 'pydantic' at runtime)
from pydantic import BaseModel
class Base(BaseModel):
"""TODO."""
class FuzzForgeWorkflowSteps(StrEnum):
"""Workflow step types."""
#: Execute a FuzzForge module
RUN_FUZZFORGE_MODULE = "run-fuzzforge-module"
class FuzzForgeWorkflowStep(Base):
"""TODO."""
#: The type of the workflow's step.
kind: FuzzForgeWorkflowSteps
class RunFuzzForgeModule(FuzzForgeWorkflowStep):
"""Execute a FuzzForge module."""
kind: Literal[FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE] = FuzzForgeWorkflowSteps.RUN_FUZZFORGE_MODULE
#: The name of the module.
module: str
#: The container of the module.
container: str
class FuzzForgeWorkflowDefinition(Base):
"""The definition of a FuzzForge workflow."""
#: The author of the workflow.
author: str
#: The identifier of the workflow.
identifier: FuzzForgeWorkflowIdentifier
#: The name of the workflow.
name: str
#: The collection of steps that compose the workflow.
steps: list[RunFuzzForgeModule]
fuzzforge-common/src/fuzzforge_common/workflows/base/parameters.py
-24
View File
@@ -1,24 +0,0 @@
from pydantic import BaseModel
from fuzzforge_common.sandboxes.engines.docker.configuration import (
DockerConfiguration, # noqa: TC001 (required by pydantic at runtime)
)
from fuzzforge_common.sandboxes.engines.podman.configuration import (
PodmanConfiguration, # noqa: TC001 (required by pydantic at runtime)
)
from fuzzforge_common.storage.configuration import StorageConfiguration # noqa: TC001 (required by pydantic at runtime)
class TemporalWorkflowParameters(BaseModel):
"""Base parameters for Temporal workflows.
Provides common configuration shared across all workflow types,
including sandbox engine and storage backend instances.
"""
#: Sandbox engine for container operations (Docker or Podman).
engine_configuration: PodmanConfiguration | DockerConfiguration
#: Storage backend for uploading/downloading execution artifacts.
storage_configuration: StorageConfiguration
fuzzforge-common/src/fuzzforge_common/workflows/bridge_utils.py
-108
View File
@@ -1,108 +0,0 @@
"""Helper utilities for working with bridge transformations."""
from pathlib import Path
from typing import Any
def load_transform_from_file(file_path: str | Path) -> str:
"""Load bridge transformation code from a Python file.
This reads the transformation function from a .py file and extracts
the code as a string suitable for the bridge module.
Args:
file_path: Path to Python file containing transform() function
Returns:
Python code as a string
Example:
>>> code = load_transform_from_file("transformations/add_line_numbers.py")
>>> # code contains the transform() function as a string
"""
path = Path(file_path)
if not path.exists():
raise FileNotFoundError(f"Transformation file not found: {file_path}")
if path.suffix != ".py":
raise ValueError(f"Transformation file must be .py file, got: {path.suffix}")
# Read the entire file
code = path.read_text()
return code
def create_bridge_input(
transform_file: str | Path,
input_filename: str | None = None,
output_filename: str | None = None,
) -> dict[str, Any]:
"""Create bridge module input configuration from a transformation file.
Args:
transform_file: Path to Python file with transform() function
input_filename: Optional specific input file to transform
output_filename: Optional specific output filename
Returns:
Dictionary suitable for bridge module's input.json
Example:
>>> config = create_bridge_input("transformations/add_line_numbers.py")
>>> import json
>>> json.dump(config, open("input.json", "w"))
"""
code = load_transform_from_file(transform_file)
return {
"code": code,
"input_filename": input_filename,
"output_filename": output_filename,
}
def validate_transform_function(file_path: str | Path) -> bool:
"""Validate that a Python file contains a valid transform() function.
Args:
file_path: Path to Python file to validate
Returns:
True if valid, raises exception otherwise
Raises:
ValueError: If transform() function is not found or invalid
"""
code = load_transform_from_file(file_path)
# Check if transform function is defined
if "def transform(" not in code:
raise ValueError(
f"File {file_path} must contain a 'def transform(data)' function"
)
# Try to compile the code
try:
compile(code, str(file_path), "exec")
except SyntaxError as e:
raise ValueError(f"Syntax error in {file_path}: {e}") from e
# Try to execute and verify transform exists
namespace: dict[str, Any] = {"__builtins__": __builtins__}
try:
exec(code, namespace)
except Exception as e:
raise ValueError(f"Failed to execute {file_path}: {e}") from e
if "transform" not in namespace:
raise ValueError(f"No 'transform' function found in {file_path}")
if not callable(namespace["transform"]):
raise ValueError(f"'transform' in {file_path} is not callable")
return True
fuzzforge-common/src/fuzzforge_common/workflows/default.py
-27
View File
@@ -1,27 +0,0 @@
from fuzzforge_types import (
FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime)
FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime)
)
from fuzzforge_common.workflows.base.definitions import (
FuzzForgeWorkflowDefinition, # noqa: TC001 (required by pydantic at runtime)
)
from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters
class ExecuteFuzzForgeWorkflowParameters(TemporalWorkflowParameters):
"""Parameters for the default FuzzForge workflow orchestration.
Contains workflow definition and execution tracking identifiers
for coordinating multi-module workflows.
"""
#: UUID7 identifier of this specific workflow execution.
execution_identifier: FuzzForgeExecutionIdentifier
#: UUID7 identifier of the project this execution belongs to.
project_identifier: FuzzForgeProjectIdentifier
#: The definition of the FuzzForge workflow to run.
workflow_definition: FuzzForgeWorkflowDefinition
fuzzforge-common/src/fuzzforge_common/workflows/modules.py
-80
View File
@@ -1,80 +0,0 @@
from typing import Any, Literal
from fuzzforge_types import (
FuzzForgeExecutionIdentifier, # noqa: TC002 (required by pydantic at runtime)
FuzzForgeProjectIdentifier, # noqa: TC002 (required by pydantic at runtime)
)
from fuzzforge_common.workflows.base.parameters import TemporalWorkflowParameters
class ExecuteFuzzForgeModuleParameters(TemporalWorkflowParameters):
"""Parameters for executing a single FuzzForge module workflow.
Contains module execution configuration including container image,
project context, and execution tracking identifiers.
Supports workflow chaining where modules can be executed in sequence,
with each module's output becoming the next module's input.
"""
#: The identifier of this module execution.
execution_identifier: FuzzForgeExecutionIdentifier
#: The identifier/name of the module to execute.
#: FIXME: Currently accepts both UUID (for registry lookups) and container names (e.g., "text-generator:0.0.1").
#: This should be split into module_identifier (UUID) and container_image (string) in the future.
module_identifier: str
#: The identifier of the project this module execution belongs to.
project_identifier: FuzzForgeProjectIdentifier
#: Optional configuration dictionary for the module.
#: Will be written to /data/input/config.json in the sandbox.
module_configuration: dict[str, Any] | None = None
# Workflow chaining fields
#: The identifier of the parent workflow execution (if part of a multi-module workflow).
#: For standalone module executions, this equals execution_identifier.
workflow_execution_identifier: FuzzForgeExecutionIdentifier | None = None
#: Position of this module in the workflow (0-based).
#: 0 = first module (reads from project assets)
#: N > 0 = subsequent module (reads from previous module's output)
step_index: int = 0
#: Execution identifier of the previous module in the workflow chain.
#: None for first module (step_index=0).
#: Used to locate previous module's output in storage.
previous_step_execution_identifier: FuzzForgeExecutionIdentifier | None = None
class WorkflowStep(TemporalWorkflowParameters):
"""A step in a workflow - a module execution.
Steps are executed sequentially in a workflow. Each step runs a containerized module.
Examples:
# Module step
WorkflowStep(
step_index=0,
step_type="module",
module_identifier="text-generator:0.0.1"
)
"""
#: Position of this step in the workflow (0-based)
step_index: int
#: Type of step: "module" (bridges are also modules now)
step_type: Literal["module"]
#: Module identifier (container image name like "text-generator:0.0.1")
#: Required if step_type="module"
module_identifier: str | None = None
#: Optional module configuration
module_configuration: dict[str, Any] | None = None