mirror of
https://github.com/FuzzingLabs/fuzzforge_ai.git
synced 2026-05-24 09:34:02 +02:00
feat: FuzzForge AI - complete rewrite for OSS release
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
# FuzzForge Modules SDK - Base image for all modules
|
||||
#
|
||||
# This image provides:
|
||||
# - Python 3.14 with uv package manager
|
||||
# - Pre-built wheels for common dependencies
|
||||
# - Standard module directory structure
|
||||
|
||||
FROM ghcr.io/astral-sh/uv:python3.14-bookworm-slim
|
||||
|
||||
# Install system dependencies commonly needed by modules
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set up application directory structure
|
||||
WORKDIR /app
|
||||
|
||||
# Create FuzzForge standard directories
|
||||
RUN mkdir -p /fuzzforge/input /fuzzforge/output
|
||||
|
||||
# Copy wheels directory (built by parent Makefile)
|
||||
COPY .wheels /wheels
|
||||
|
||||
# Set up uv for the container
|
||||
ENV UV_SYSTEM_PYTHON=1
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
|
||||
# Default entrypoint - modules override this
|
||||
ENTRYPOINT ["uv", "run", "module"]
|
||||
@@ -0,0 +1,39 @@
|
||||
PACKAGE=$(word 1, $(shell uv version))
|
||||
VERSION=$(word 2, $(shell uv version))
|
||||
|
||||
SOURCES=./src
|
||||
TESTS=./tests
|
||||
|
||||
FUZZFORGE_MODULE_TEMPLATE=$(PWD)/src/fuzzforge_modules_sdk/templates/module
|
||||
|
||||
.PHONY: bandit clean format mypy pytest ruff version
|
||||
|
||||
bandit:
|
||||
uv run bandit --recursive $(SOURCES)
|
||||
|
||||
clean:
|
||||
@find . -type d \( \
|
||||
-name '*.egg-info' \
|
||||
-o -name '.mypy_cache' \
|
||||
-o -name '.pytest_cache' \
|
||||
-o -name '.ruff_cache' \
|
||||
-o -name '__pycache__' \
|
||||
\) -printf 'removing directory %p\n' -exec rm -rf {} +
|
||||
|
||||
cloc:
|
||||
cloc $(SOURCES)
|
||||
|
||||
format:
|
||||
uv run ruff format $(SOURCES) $(TESTS)
|
||||
|
||||
mypy:
|
||||
uv run mypy $(SOURCES)
|
||||
|
||||
pytest:
|
||||
uv run pytest $(TESTS)
|
||||
|
||||
ruff:
|
||||
uv run ruff check --fix $(SOURCES) $(TESTS)
|
||||
|
||||
version:
|
||||
@echo '$(PACKAGE)@$(VERSION)'
|
||||
@@ -0,0 +1,67 @@
|
||||
# FuzzForge Modules SDK
|
||||
|
||||
...
|
||||
|
||||
# Setup
|
||||
|
||||
- start the podman user socket
|
||||
|
||||
```shell
|
||||
systemctl --user start podman.socket
|
||||
```
|
||||
|
||||
NB : you can also automaticllay start it at boot
|
||||
|
||||
```shell
|
||||
systemctl --user enable --now podman.socket
|
||||
```
|
||||
|
||||
## HACK : fix missing `fuzzforge-modules-sdk`
|
||||
|
||||
- if you have this error when using some fuzzforge-modules-sdk deps :
|
||||
|
||||
```shell
|
||||
❯ make format
|
||||
uv run ruff format ./src ./tests
|
||||
× No solution found when resolving dependencies:
|
||||
╰─▶ Because fuzzforge-modules-sdk was not found in the package registry and your project depends on fuzzforge-modules-sdk==0.0.1, we can
|
||||
conclude that your project's requirements are unsatisfiable.
|
||||
And because your project requires opengrep[lints], we can conclude that your project's requirements are unsatisfiable.
|
||||
make: *** [Makefile:30: format] Error 1
|
||||
```
|
||||
|
||||
- build a wheel package of fuzzforge-modules-sdk
|
||||
|
||||
```shell
|
||||
cd fuzzforge_ng/fuzzforge-modules/fuzzforge-modules-sdk
|
||||
uv build
|
||||
```
|
||||
|
||||
- then inside your module project, install it
|
||||
|
||||
```shell
|
||||
cd fuzzforge_ng_modules/mymodule
|
||||
uv sync --all-extras --find-links ../../fuzzforge_ng/dist/
|
||||
```
|
||||
|
||||
# Usage
|
||||
|
||||
## Prepare
|
||||
|
||||
- enter venv (or use uv run)
|
||||
|
||||
```shell
|
||||
source .venv/bin/activate
|
||||
```
|
||||
|
||||
- create a new module
|
||||
|
||||
```shell
|
||||
fuzzforge-modules-sdk new module --name my_new_module --directory ../fuzzforge_ng_modules/
|
||||
```
|
||||
|
||||
- build the base image
|
||||
|
||||
```shell
|
||||
fuzzforge-modules-sdk build image
|
||||
```
|
||||
@@ -0,0 +1,7 @@
|
||||
[mypy]
|
||||
exclude = ^src/fuzzforge_modules_sdk/templates/.*
|
||||
plugins = pydantic.mypy
|
||||
strict = True
|
||||
warn_unused_ignores = True
|
||||
warn_redundant_casts = True
|
||||
warn_return_any = True
|
||||
@@ -0,0 +1,31 @@
|
||||
[project]
|
||||
name = "fuzzforge-modules-sdk"
|
||||
version = "0.0.1"
|
||||
description = "Software development kit (SDK) for FuzzForge's modules."
|
||||
authors = []
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"podman==5.6.0",
|
||||
"pydantic==2.12.4",
|
||||
"tomlkit==0.13.3",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
lints = [
|
||||
"bandit==1.8.6",
|
||||
"mypy==1.18.2",
|
||||
"ruff==0.14.4",
|
||||
]
|
||||
tests = [
|
||||
"pytest==9.0.2",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
fuzzforge-modules-sdk = "fuzzforge_modules_sdk._cli.main:main"
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
fuzzforge_modules_sdk = [
|
||||
"assets/**/*",
|
||||
"templates/**/*",
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
line-length = 120
|
||||
|
||||
[lint]
|
||||
select = [ "ALL" ]
|
||||
ignore = [
|
||||
"COM812", # conflicts with the formatter
|
||||
"D100", # ignoring missing docstrings in public modules
|
||||
"D104", # ignoring missing docstrings in public packages
|
||||
"D203", # conflicts with 'D211'
|
||||
"D213", # conflicts with 'D212'
|
||||
"TD002", # ignoring missing author in 'TODO' statements
|
||||
"TD003", # ignoring missing issue link in 'TODO' statements
|
||||
]
|
||||
|
||||
[lint.per-file-ignores]
|
||||
"tests/*" = [
|
||||
"PLR2004", # allowing comparisons using unamed numerical constants in tests
|
||||
"S101", # allowing 'assert' statements in tests
|
||||
]
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from shutil import copyfile, copytree
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
import os
|
||||
|
||||
from podman import PodmanClient
|
||||
from tomlkit import TOMLDocument, parse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from importlib.resources.abc import Traversable
|
||||
|
||||
|
||||
def _get_default_podman_socket() -> str:
|
||||
"""Get the default Podman socket path for the current user."""
|
||||
uid = os.getuid()
|
||||
return f"unix:///run/user/{uid}/podman/podman.sock"
|
||||
|
||||
|
||||
PATH_TO_SOURCES: Path = Path(__file__).parent.parent
|
||||
|
||||
|
||||
def _build_podman_image(directory: Path, tag: str, socket: str | None = None) -> None:
|
||||
if socket is None:
|
||||
socket = _get_default_podman_socket()
|
||||
with PodmanClient(base_url=socket) as client:
|
||||
client.images.build(
|
||||
dockerfile="Dockerfile",
|
||||
nocache=True,
|
||||
path=directory,
|
||||
tag=tag,
|
||||
)
|
||||
|
||||
|
||||
def build_base_image(engine: Literal["podman"], socket: str | None = None) -> None:
|
||||
with TemporaryDirectory() as directory:
|
||||
path_to_assets: Traversable = files("fuzzforge_modules_sdk").joinpath("assets")
|
||||
copyfile(
|
||||
src=str(path_to_assets.joinpath("Dockerfile")),
|
||||
dst=Path(directory).joinpath("Dockerfile"),
|
||||
)
|
||||
copyfile(
|
||||
src=str(path_to_assets.joinpath("pyproject.toml")),
|
||||
dst=Path(directory).joinpath("pyproject.toml"),
|
||||
)
|
||||
copytree(src=str(PATH_TO_SOURCES), dst=Path(directory).joinpath("src").joinpath(PATH_TO_SOURCES.name))
|
||||
|
||||
# update the file 'pyproject.toml'
|
||||
path: Path = Path(directory).joinpath("pyproject.toml")
|
||||
data: TOMLDocument = parse(path.read_text())
|
||||
name: str = data["project"]["name"] # type: ignore[assignment, index]
|
||||
version: str = data["project"]["version"] # type: ignore[assignment, index]
|
||||
tag: str = f"{name}:{version}"
|
||||
|
||||
match engine:
|
||||
case "podman":
|
||||
_build_podman_image(
|
||||
directory=Path(directory),
|
||||
socket=socket,
|
||||
tag=tag,
|
||||
)
|
||||
case _:
|
||||
message: str = f"unsupported engine '{engine}'"
|
||||
raise Exception(message) # noqa: TRY002
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib.resources import files
|
||||
from shutil import copytree, ignore_patterns
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from tomlkit import dumps, parse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from importlib.resources.abc import Traversable
|
||||
from pathlib import Path
|
||||
|
||||
from tomlkit import TOMLDocument
|
||||
|
||||
|
||||
def create_new_module(name: str, directory: Path) -> None:
|
||||
source: Traversable = files("fuzzforge_modules_sdk").joinpath("templates").joinpath("fuzzforge-module-template")
|
||||
destination: Path = directory.joinpath(name) # TODO: sanitize path
|
||||
copytree(
|
||||
src=str(source),
|
||||
dst=destination,
|
||||
ignore=ignore_patterns("__pycache__", "*.egg-info", "*.pyc", ".mypy_cache", ".ruff_cache", ".venv"),
|
||||
)
|
||||
|
||||
# update the file 'pyproject.toml'
|
||||
path: Path = destination.joinpath("pyproject.toml")
|
||||
data: TOMLDocument = parse(path.read_text())
|
||||
data["project"]["name"] = name # type: ignore[index]
|
||||
del data["tool"]["uv"]["sources"] # type: ignore[index, union-attr]
|
||||
path.write_text(dumps(data))
|
||||
@@ -0,0 +1,71 @@
|
||||
from argparse import ArgumentParser
|
||||
from pathlib import Path
|
||||
|
||||
from fuzzforge_modules_sdk._cli.build_base_image import build_base_image
|
||||
from fuzzforge_modules_sdk._cli.create_new_module import create_new_module
|
||||
|
||||
|
||||
def create_parser() -> ArgumentParser:
|
||||
parser: ArgumentParser = ArgumentParser(
|
||||
prog="fuzzforge-modules-sdk", description="Utilities for the Fuzzforge Modules SDK."
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(required=True)
|
||||
|
||||
# fuzzforge-modules-sdk build ...
|
||||
parser_build = subparsers.add_parser(name="build")
|
||||
|
||||
subparsers_build = parser_build.add_subparsers(required=True)
|
||||
|
||||
# fuzzforge-modules-sdk build image ...
|
||||
parser_build_image = subparsers_build.add_parser(
|
||||
name="image",
|
||||
help="Build the image.",
|
||||
)
|
||||
parser_build_image.add_argument(
|
||||
"--engine",
|
||||
default="podman",
|
||||
)
|
||||
parser_build_image.add_argument(
|
||||
"--socket",
|
||||
default=None,
|
||||
)
|
||||
parser_build_image.set_defaults(
|
||||
function_to_execute=build_base_image,
|
||||
)
|
||||
|
||||
# fuzzforge-modules-sdk new ...
|
||||
parser_new = subparsers.add_parser(name="new")
|
||||
|
||||
subparsers_new = parser_new.add_subparsers(required=True)
|
||||
|
||||
# fuzzforge-modules-sdk new module ...
|
||||
parser_new_module = subparsers_new.add_parser(
|
||||
name="module",
|
||||
help="Generate the boilerplate required to create a new module.",
|
||||
)
|
||||
parser_new_module.add_argument(
|
||||
"--name",
|
||||
help="The name of the module to create.",
|
||||
required=True,
|
||||
)
|
||||
parser_new_module.add_argument(
|
||||
"--directory",
|
||||
default=".",
|
||||
type=Path,
|
||||
help="The directory the new module should be created into (defaults to current working directory).",
|
||||
)
|
||||
parser_new_module.set_defaults(
|
||||
function_to_execute=create_new_module,
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Entry point for the command-line interface."""
|
||||
parser: ArgumentParser = create_parser()
|
||||
arguments = parser.parse_args()
|
||||
function_to_execute = arguments.function_to_execute
|
||||
del arguments.function_to_execute
|
||||
function_to_execute(**vars(arguments))
|
||||
@@ -0,0 +1,13 @@
|
||||
from pathlib import Path
|
||||
|
||||
PATH_TO_DATA: Path = Path("/data")
|
||||
PATH_TO_INPUTS: Path = PATH_TO_DATA.joinpath("input")
|
||||
PATH_TO_INPUT: Path = PATH_TO_INPUTS.joinpath("input.json")
|
||||
PATH_TO_OUTPUTS: Path = PATH_TO_DATA.joinpath("output")
|
||||
PATH_TO_ARTIFACTS: Path = PATH_TO_OUTPUTS.joinpath("artifacts")
|
||||
PATH_TO_RESULTS: Path = PATH_TO_OUTPUTS.joinpath("results.json")
|
||||
PATH_TO_LOGS: Path = PATH_TO_OUTPUTS.joinpath("logs.jsonl")
|
||||
|
||||
# Streaming output paths for real-time progress
|
||||
PATH_TO_PROGRESS: Path = PATH_TO_OUTPUTS.joinpath("progress.json")
|
||||
PATH_TO_STREAM: Path = PATH_TO_OUTPUTS.joinpath("stream.jsonl")
|
||||
@@ -0,0 +1,2 @@
|
||||
class FuzzForgeModuleError(Exception):
|
||||
"""TODO."""
|
||||
@@ -0,0 +1,43 @@
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import structlog
|
||||
|
||||
from fuzzforge_modules_sdk.api.constants import PATH_TO_LOGS
|
||||
|
||||
|
||||
class Formatter(logging.Formatter):
|
||||
"""TODO."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
"""TODO."""
|
||||
record.exc_info = None
|
||||
return super().format(record)
|
||||
|
||||
|
||||
def configure() -> None:
|
||||
"""TODO."""
|
||||
fmt: str = "%(message)s"
|
||||
level = logging.DEBUG
|
||||
PATH_TO_LOGS.parent.mkdir(exist_ok=True, parents=True)
|
||||
PATH_TO_LOGS.unlink(missing_ok=True)
|
||||
handler_file = logging.FileHandler(filename=PATH_TO_LOGS, mode="a")
|
||||
handler_file.setFormatter(fmt=Formatter(fmt=fmt))
|
||||
handler_file.setLevel(level=level)
|
||||
handler_stderr = logging.StreamHandler(stream=sys.stderr)
|
||||
handler_stderr.setFormatter(fmt=Formatter(fmt=fmt))
|
||||
handler_stderr.setLevel(level=level)
|
||||
logger: logging.Logger = logging.getLogger()
|
||||
logger.setLevel(level=level)
|
||||
logger.addHandler(handler_file)
|
||||
logger.addHandler(handler_stderr)
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.stdlib.add_log_level,
|
||||
structlog.processors.TimeStamper(fmt="iso"),
|
||||
structlog.processors.dict_tracebacks,
|
||||
structlog.processors.JSONRenderer(),
|
||||
],
|
||||
logger_factory=structlog.stdlib.LoggerFactory(),
|
||||
wrapper_class=structlog.stdlib.BoundLogger,
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
from enum import StrEnum
|
||||
from pathlib import Path # noqa: TC003 (required by pydantic at runtime)
|
||||
from typing import TypeVar
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class Base(BaseModel):
|
||||
"""TODO."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class FuzzForgeModulesSettingsBase(Base):
|
||||
"""TODO."""
|
||||
|
||||
|
||||
FuzzForgeModulesSettingsType = TypeVar("FuzzForgeModulesSettingsType", bound=FuzzForgeModulesSettingsBase)
|
||||
|
||||
|
||||
class FuzzForgeModuleResources(StrEnum):
|
||||
"""Enumeration of artifact types."""
|
||||
|
||||
#: The type of the resource is unknown or irrelevant.
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class FuzzForgeModuleResource(Base):
|
||||
"""TODO."""
|
||||
|
||||
#: The description of the resource.
|
||||
description: str
|
||||
#: The type of the resource.
|
||||
kind: FuzzForgeModuleResources
|
||||
#: The name of the resource.
|
||||
name: str
|
||||
#: The path of the resource on disk.
|
||||
path: Path
|
||||
|
||||
|
||||
class FuzzForgeModuleInputBase[FuzzForgeModulesSettingsType: FuzzForgeModulesSettingsBase](Base):
|
||||
"""The (standardized) input of a FuzzForge module."""
|
||||
|
||||
#: The collection of resources given to the module as inputs.
|
||||
resources: list[FuzzForgeModuleResource]
|
||||
#: The settings of the module.
|
||||
settings: FuzzForgeModulesSettingsType
|
||||
|
||||
|
||||
class FuzzForgeModuleArtifacts(StrEnum):
|
||||
"""Enumeration of artifact types."""
|
||||
|
||||
#: The artifact is an asset.
|
||||
ASSET = "asset"
|
||||
|
||||
|
||||
class FuzzForgeModuleArtifact(Base):
|
||||
"""An artifact generated by the module during its run."""
|
||||
|
||||
#: The description of the artifact.
|
||||
description: str
|
||||
#: The type of the artifact.
|
||||
kind: FuzzForgeModuleArtifacts
|
||||
#: The name of the artifact.
|
||||
name: str
|
||||
#: The path to the artifact on disk.
|
||||
path: Path
|
||||
|
||||
|
||||
class FuzzForgeModuleResults(StrEnum):
|
||||
"""TODO."""
|
||||
|
||||
SUCCESS = "success"
|
||||
FAILURE = "failure"
|
||||
|
||||
|
||||
class FuzzForgeModuleOutputBase(Base):
|
||||
"""The (standardized) output of a FuzzForge module."""
|
||||
|
||||
#: The collection of artifacts generated by the module during its run.
|
||||
artifacts: list[FuzzForgeModuleArtifacts]
|
||||
#: The path to the logs.
|
||||
logs: Path
|
||||
#: The result of the module's run.
|
||||
result: FuzzForgeModuleResults
|
||||
+288
@@ -0,0 +1,288 @@
|
||||
from abc import ABC, abstractmethod
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from shutil import rmtree
|
||||
from typing import TYPE_CHECKING, Any, Final, final
|
||||
|
||||
from structlog import get_logger
|
||||
|
||||
from fuzzforge_modules_sdk.api.constants import (
|
||||
PATH_TO_ARTIFACTS,
|
||||
PATH_TO_INPUT,
|
||||
PATH_TO_LOGS,
|
||||
PATH_TO_PROGRESS,
|
||||
PATH_TO_RESULTS,
|
||||
PATH_TO_STREAM,
|
||||
)
|
||||
from fuzzforge_modules_sdk.api.exceptions import FuzzForgeModuleError
|
||||
from fuzzforge_modules_sdk.api.models import (
|
||||
FuzzForgeModuleArtifact,
|
||||
FuzzForgeModuleArtifacts,
|
||||
FuzzForgeModuleInputBase,
|
||||
FuzzForgeModuleOutputBase,
|
||||
FuzzForgeModuleResource,
|
||||
FuzzForgeModuleResults,
|
||||
FuzzForgeModulesSettingsType,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from structlog.stdlib import BoundLogger
|
||||
|
||||
|
||||
class FuzzForgeModule(ABC):
|
||||
"""FuzzForge Modules' base."""
|
||||
|
||||
__artifacts: dict[str, FuzzForgeModuleArtifact]
|
||||
|
||||
#: The logger associated with the module.
|
||||
__logger: Final[BoundLogger]
|
||||
|
||||
#: The name of the module.
|
||||
__name: Final[str]
|
||||
|
||||
#: The version of the module.
|
||||
__version: Final[str]
|
||||
|
||||
#: Start time for progress tracking.
|
||||
__start_time: float
|
||||
|
||||
#: Custom output data set by the module.
|
||||
__output_data: dict[str, Any]
|
||||
|
||||
def __init__(self, name: str, version: str) -> None:
|
||||
"""Initialize an instance of the class.
|
||||
|
||||
:param name: The name of the module.
|
||||
:param version: The version of the module.
|
||||
|
||||
"""
|
||||
self.__artifacts = {}
|
||||
self.__logger = get_logger("module")
|
||||
self.__name = name
|
||||
self.__version = version
|
||||
self.__start_time = time.time()
|
||||
self.__output_data = {}
|
||||
|
||||
# Initialize streaming output files
|
||||
PATH_TO_PROGRESS.parent.mkdir(exist_ok=True, parents=True)
|
||||
PATH_TO_STREAM.parent.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
@final
|
||||
def get_logger(self) -> BoundLogger:
|
||||
"""Return the logger associated with the module."""
|
||||
return self.__logger
|
||||
|
||||
@final
|
||||
def get_name(self) -> str:
|
||||
"""Return the name of the module."""
|
||||
return self.__name
|
||||
|
||||
@final
|
||||
def get_version(self) -> str:
|
||||
"""Return the version of the module."""
|
||||
return self.__version
|
||||
|
||||
@final
|
||||
def set_output(self, **kwargs: Any) -> None:
|
||||
"""Set custom output data to be included in results.json.
|
||||
|
||||
Call this from _run() to add module-specific fields to the output.
|
||||
|
||||
:param kwargs: Key-value pairs to include in the output.
|
||||
|
||||
Example:
|
||||
self.set_output(
|
||||
total_targets=4,
|
||||
valid_targets=["target1", "target2"],
|
||||
results=[...]
|
||||
)
|
||||
|
||||
"""
|
||||
self.__output_data.update(kwargs)
|
||||
|
||||
@final
|
||||
def emit_progress(
|
||||
self,
|
||||
progress: int,
|
||||
status: str = "running",
|
||||
message: str = "",
|
||||
metrics: dict[str, Any] | None = None,
|
||||
current_task: str = "",
|
||||
) -> None:
|
||||
"""Emit a progress update to the progress file.
|
||||
|
||||
This method writes to /data/output/progress.json which can be polled
|
||||
by the orchestrator or UI to show real-time progress.
|
||||
|
||||
:param progress: Progress percentage (0-100).
|
||||
:param status: Current status ("initializing", "running", "completed", "failed").
|
||||
:param message: Human-readable status message.
|
||||
:param metrics: Dictionary of metrics (e.g., {"executions": 1000, "coverage": 50}).
|
||||
:param current_task: Name of the current task being performed.
|
||||
|
||||
"""
|
||||
elapsed = time.time() - self.__start_time
|
||||
|
||||
progress_data = {
|
||||
"module": self.__name,
|
||||
"version": self.__version,
|
||||
"status": status,
|
||||
"progress": max(0, min(100, progress)),
|
||||
"message": message,
|
||||
"current_task": current_task,
|
||||
"elapsed_seconds": round(elapsed, 2),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"metrics": metrics or {},
|
||||
}
|
||||
|
||||
PATH_TO_PROGRESS.write_text(json.dumps(progress_data, indent=2))
|
||||
|
||||
@final
|
||||
def emit_event(self, event: str, **data: Any) -> None:
|
||||
"""Emit a streaming event to the stream file.
|
||||
|
||||
This method appends to /data/output/stream.jsonl which can be tailed
|
||||
by the orchestrator or UI for real-time event streaming.
|
||||
|
||||
:param event: Event type (e.g., "crash_found", "target_started", "metrics").
|
||||
:param data: Additional event data as keyword arguments.
|
||||
|
||||
"""
|
||||
elapsed = time.time() - self.__start_time
|
||||
|
||||
event_data = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"elapsed_seconds": round(elapsed, 2),
|
||||
"module": self.__name,
|
||||
"event": event,
|
||||
**data,
|
||||
}
|
||||
|
||||
# Append to stream file (create if doesn't exist)
|
||||
with PATH_TO_STREAM.open("a") as f:
|
||||
f.write(json.dumps(event_data) + "\n")
|
||||
|
||||
@final
|
||||
def get_elapsed_seconds(self) -> float:
|
||||
"""Return the elapsed time since module start.
|
||||
|
||||
:returns: Elapsed time in seconds.
|
||||
|
||||
"""
|
||||
return time.time() - self.__start_time
|
||||
|
||||
@final
|
||||
def _register_artifact(self, name: str, kind: FuzzForgeModuleArtifacts, description: str, path: Path) -> None:
|
||||
"""Register an artifact.
|
||||
|
||||
:param name: The name of the artifact.
|
||||
:param kind: The type of the artifact.
|
||||
:param description: The description of the artifact.
|
||||
:param path: The path of the artifact on the file system.
|
||||
|
||||
"""
|
||||
source: Path = path.resolve(strict=True)
|
||||
destination: Path = PATH_TO_ARTIFACTS.joinpath(name).resolve()
|
||||
if destination.parent != PATH_TO_ARTIFACTS:
|
||||
message: str = f"path '{destination} is not a direct descendant of path '{PATH_TO_ARTIFACTS}'"
|
||||
raise FuzzForgeModuleError(message)
|
||||
if destination.exists(follow_symlinks=False):
|
||||
if destination.is_file() or destination.is_symlink():
|
||||
destination.unlink()
|
||||
elif destination.is_dir():
|
||||
rmtree(destination)
|
||||
else:
|
||||
message = f"unable to remove resource at path '{destination}': unsupported resource type"
|
||||
raise FuzzForgeModuleError(message)
|
||||
destination.parent.mkdir(exist_ok=True, parents=True)
|
||||
source.copy(destination)
|
||||
self.__artifacts[name] = FuzzForgeModuleArtifact(
|
||||
description=description,
|
||||
kind=kind,
|
||||
name=name,
|
||||
path=path,
|
||||
)
|
||||
|
||||
@final
|
||||
def main(self) -> None:
|
||||
"""TODO."""
|
||||
result = FuzzForgeModuleResults.SUCCESS
|
||||
|
||||
try:
|
||||
buffer: bytes = PATH_TO_INPUT.read_bytes()
|
||||
data = self._get_input_type().model_validate_json(buffer)
|
||||
self._prepare(settings=data.settings)
|
||||
except: # noqa: E722
|
||||
self.get_logger().exception(event="exception during 'prepare' step")
|
||||
result = FuzzForgeModuleResults.FAILURE
|
||||
|
||||
if result != FuzzForgeModuleResults.FAILURE:
|
||||
try:
|
||||
result = self._run(resources=data.resources)
|
||||
except: # noqa: E722
|
||||
self.get_logger().exception(event="exception during 'run' step")
|
||||
result = FuzzForgeModuleResults.FAILURE
|
||||
|
||||
if result != FuzzForgeModuleResults.FAILURE:
|
||||
try:
|
||||
self._cleanup(settings=data.settings)
|
||||
except: # noqa: E722
|
||||
self.get_logger().exception(event="exception during 'cleanup' step")
|
||||
|
||||
output = self._get_output_type()(
|
||||
artifacts=list(self.__artifacts.values()),
|
||||
logs=PATH_TO_LOGS,
|
||||
result=result,
|
||||
**self.__output_data,
|
||||
)
|
||||
buffer = output.model_dump_json().encode("utf-8")
|
||||
PATH_TO_RESULTS.parent.mkdir(exist_ok=True, parents=True)
|
||||
PATH_TO_RESULTS.write_bytes(buffer)
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def _get_input_type(cls) -> type[FuzzForgeModuleInputBase[Any]]:
|
||||
"""TODO."""
|
||||
message: str = f"method '_get_input_type' is not implemented for class '{cls.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def _get_output_type(cls) -> type[FuzzForgeModuleOutputBase]:
|
||||
"""TODO."""
|
||||
message: str = f"method '_get_output_type' is not implemented for class '{cls.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def _prepare(self, settings: FuzzForgeModulesSettingsType) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param settings: TODO.
|
||||
|
||||
"""
|
||||
message: str = f"method '_prepare' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults:
|
||||
"""TODO.
|
||||
|
||||
:param resources: TODO.
|
||||
:returns: TODO.
|
||||
|
||||
"""
|
||||
message: str = f"method '_run' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
|
||||
@abstractmethod
|
||||
def _cleanup(self, settings: FuzzForgeModulesSettingsType) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param settings: TODO.
|
||||
|
||||
"""
|
||||
message: str = f"method '_cleanup' is not implemented for class '{self.__class__.__name__}'"
|
||||
raise NotImplementedError(message)
|
||||
@@ -0,0 +1,20 @@
|
||||
FROM docker.io/debian:trixie as base
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.9.10 /uv /uvx /bin/
|
||||
|
||||
FROM base as builder
|
||||
|
||||
WORKDIR /sdk
|
||||
|
||||
COPY ./src /sdk/src
|
||||
COPY ./pyproject.toml /sdk/pyproject.toml
|
||||
|
||||
RUN uv build --wheel -o /sdk/distributions
|
||||
|
||||
FROM base as final
|
||||
|
||||
COPY --from=builder /sdk/distributions /wheels
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
CMD [ "/usr/bin/sleep", "infinity" ]
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../../pyproject.toml
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
FROM localhost/fuzzforge-modules-sdk:0.0.1
|
||||
|
||||
COPY ./src /app/src
|
||||
COPY ./pyproject.toml /app/pyproject.toml
|
||||
|
||||
# Remove workspace reference since we're using wheels
|
||||
RUN sed -i '/\[tool\.uv\.sources\]/,/^$/d' /app/pyproject.toml
|
||||
|
||||
RUN uv sync --find-links /wheels
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
PACKAGE=$(word 1, $(shell uv version))
|
||||
VERSION=$(word 2, $(shell uv version))
|
||||
|
||||
PODMAN?=/usr/bin/podman
|
||||
|
||||
SOURCES=./src
|
||||
TESTS=./tests
|
||||
|
||||
.PHONY: bandit build clean format mypy pytest ruff version
|
||||
|
||||
bandit:
|
||||
uv run bandit --recursive $(SOURCES)
|
||||
|
||||
build:
|
||||
$(PODMAN) build --file ./Dockerfile --no-cache --tag $(PACKAGE):$(VERSION)
|
||||
|
||||
save: build
|
||||
$(PODMAN) save --format oci-archive --output /tmp/$(PACKAGE)-$(VERSION).oci $(PACKAGE):$(VERSION)
|
||||
|
||||
clean:
|
||||
@find . -type d \( \
|
||||
-name '*.egg-info' \
|
||||
-o -name '.mypy_cache' \
|
||||
-o -name '.pytest_cache' \
|
||||
-o -name '.ruff_cache' \
|
||||
-o -name '__pycache__' \
|
||||
\) -printf 'removing directory %p\n' -exec rm -rf {} +
|
||||
|
||||
cloc:
|
||||
cloc $(SOURCES)
|
||||
|
||||
format:
|
||||
uv run ruff format $(SOURCES) $(TESTS)
|
||||
|
||||
mypy:
|
||||
uv run mypy $(SOURCES)
|
||||
|
||||
pytest:
|
||||
uv run pytest $(TESTS)
|
||||
|
||||
ruff:
|
||||
uv run ruff check --fix $(SOURCES) $(TESTS)
|
||||
|
||||
version:
|
||||
@echo '$(PACKAGE)@$(VERSION)'
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
# FuzzForge Modules - FIXME
|
||||
|
||||
## Installation
|
||||
|
||||
### Python
|
||||
|
||||
```shell
|
||||
# install the package (users)
|
||||
uv sync
|
||||
# install the package and all development dependencies (developers)
|
||||
uv sync --all-extras
|
||||
```
|
||||
|
||||
### Container
|
||||
|
||||
```shell
|
||||
# build the image
|
||||
make build
|
||||
# run the container
|
||||
mkdir -p "${PWD}/data" "${PWD}/data/input" "${PWD}/data/output"
|
||||
echo '{"settings":{},"resources":[]}' > "${PWD}/data/input/input.json"
|
||||
podman run --rm \
|
||||
--volume "${PWD}/data:/data" \
|
||||
'<name>:<version>' 'uv run module'
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```shell
|
||||
uv run module
|
||||
```
|
||||
|
||||
## Development tools
|
||||
|
||||
```shell
|
||||
# run ruff (formatter)
|
||||
make format
|
||||
# run mypy (type checker)
|
||||
make mypy
|
||||
# run tests (pytest)
|
||||
make pytest
|
||||
# run ruff (linter)
|
||||
make ruff
|
||||
```
|
||||
|
||||
See the file `Makefile` at the root of this directory for more tools.
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
[mypy]
|
||||
plugins = pydantic.mypy
|
||||
strict = True
|
||||
warn_unused_ignores = True
|
||||
warn_redundant_casts = True
|
||||
warn_return_any = True
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
[project]
|
||||
name = "fuzzforge-module-template"
|
||||
version = "0.0.1"
|
||||
description = "FIXME"
|
||||
authors = []
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
"fuzzforge-modules-sdk==0.0.1",
|
||||
"pydantic==2.12.4",
|
||||
"structlog==25.5.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
lints = [
|
||||
"bandit==1.8.6",
|
||||
"mypy==1.18.2",
|
||||
"ruff==0.14.4",
|
||||
]
|
||||
tests = [
|
||||
"pytest==9.0.2",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
module = "module.__main__:main"
|
||||
|
||||
[tool.uv.sources]
|
||||
fuzzforge-modules-sdk = { workspace = true }
|
||||
|
||||
[tool.uv]
|
||||
package = true
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
line-length = 120
|
||||
|
||||
[lint]
|
||||
select = [ "ALL" ]
|
||||
ignore = [
|
||||
"COM812", # conflicts with the formatter
|
||||
"D100", # ignoring missing docstrings in public modules
|
||||
"D104", # ignoring missing docstrings in public packages
|
||||
"D203", # conflicts with 'D211'
|
||||
"D213", # conflicts with 'D212'
|
||||
"TD002", # ignoring missing author in 'TODO' statements
|
||||
"TD003", # ignoring missing issue link in 'TODO' statements
|
||||
]
|
||||
|
||||
[lint.per-file-ignores]
|
||||
"tests/*" = [
|
||||
"PLR2004", # allowing comparisons using unamed numerical constants in tests
|
||||
"S101", # allowing 'assert' statements in tests
|
||||
]
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fuzzforge_modules_sdk.api import logs
|
||||
|
||||
from module.mod import Module
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""TODO."""
|
||||
logs.configure()
|
||||
module: FuzzForgeModule = Module()
|
||||
module.main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResults
|
||||
from fuzzforge_modules_sdk.api.modules.base import FuzzForgeModule
|
||||
|
||||
from module.models import Input, Output
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fuzzforge_modules_sdk.api.models import FuzzForgeModuleResource, FuzzForgeModulesSettingsType
|
||||
|
||||
|
||||
class Module(FuzzForgeModule):
|
||||
"""TODO."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize an instance of the class."""
|
||||
name: str = "FIXME"
|
||||
version: str = "FIXME"
|
||||
FuzzForgeModule.__init__(self, name=name, version=version)
|
||||
|
||||
@classmethod
|
||||
def _get_input_type(cls) -> type[Input]:
|
||||
"""TODO."""
|
||||
return Input
|
||||
|
||||
@classmethod
|
||||
def _get_output_type(cls) -> type[Output]:
|
||||
"""TODO."""
|
||||
return Output
|
||||
|
||||
def _prepare(self, settings: FuzzForgeModulesSettingsType) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param settings: TODO.
|
||||
|
||||
"""
|
||||
|
||||
def _run(self, resources: list[FuzzForgeModuleResource]) -> FuzzForgeModuleResults: # noqa: ARG002
|
||||
"""TODO.
|
||||
|
||||
:param resources: TODO.
|
||||
:returns: TODO.
|
||||
|
||||
"""
|
||||
return FuzzForgeModuleResults.SUCCESS
|
||||
|
||||
def _cleanup(self, settings: FuzzForgeModulesSettingsType) -> None:
|
||||
"""TODO.
|
||||
|
||||
:param settings: TODO.
|
||||
|
||||
"""
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
from fuzzforge_modules_sdk.api.models import FuzzForgeModuleInputBase, FuzzForgeModuleOutputBase
|
||||
|
||||
from module.settings import Settings
|
||||
|
||||
|
||||
class Input(FuzzForgeModuleInputBase[Settings]):
|
||||
"""TODO."""
|
||||
|
||||
|
||||
class Output(FuzzForgeModuleOutputBase):
|
||||
"""TODO."""
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
from fuzzforge_modules_sdk.api.models import FuzzForgeModulesSettingsBase
|
||||
|
||||
|
||||
class Settings(FuzzForgeModulesSettingsBase):
|
||||
"""TODO."""
|
||||
|
||||
# Here goes your attributes
|
||||
Reference in New Issue
Block a user