CalvinBackup/fuzzforge_ai
1
0
Fork 0
mirror of https://github.com/FuzzingLabs/fuzzforge_ai.git synced 2026-07-20 11:00:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

rename: FuzzForge → SecPipe

Browse Source 
Rename the entire project from FuzzForge to SecPipe:
- Python packages: fuzzforge_cli → secpipe_cli, fuzzforge_common → secpipe_common,
  fuzzforge_mcp → secpipe_mcp, fuzzforge_tests → secpipe_tests
- Directories: fuzzforge-cli → secpipe-cli, fuzzforge-common → secpipe-common,
  fuzzforge-mcp → secpipe-mcp, fuzzforge-tests → secpipe-tests
- Environment variables: FUZZFORGE_* → SECPIPE_*
- MCP server name: SecPipe MCP Server
- CI workflows, Makefile, Dockerfile, hub-config, NOTICE updated
- Fix mcp-server.yml to use uvicorn secpipe_mcp.application:app
This commit is contained in:
AFredefon
2026-04-09 04:10:46 +02:00
parent bbf864e88b
commit be009a4094
120 changed files with 866 additions and 800 deletions
Download Patch File Download Diff File Expand all files Collapse all files
secpipe-common/src/secpipe_common/__init__.py
+38
View File
@@ -0,0 +1,38 @@
"""SecPipe Common - Shared abstractions and implementations for SecPipe.
This package provides:
- Sandbox engine abstractions (Podman, Docker)
- Common exceptions
Example usage:
from secpipe_common import (
AbstractSecPipeSandboxEngine,
ImageInfo,
Podman,
PodmanConfiguration,
)
"""
from secpipe_common.exceptions import SecPipeError
from secpipe_common.sandboxes import (
AbstractSecPipeEngineConfiguration,
AbstractSecPipeSandboxEngine,
Docker,
DockerConfiguration,
SecPipeSandboxEngines,
ImageInfo,
Podman,
PodmanConfiguration,
)
__all__ = [
"AbstractSecPipeEngineConfiguration",
"AbstractSecPipeSandboxEngine",
"Docker",
"DockerConfiguration",
"SecPipeError",
"SecPipeSandboxEngines",
"ImageInfo",
"Podman",
"PodmanConfiguration",
]
secpipe-common/src/secpipe_common/exceptions.py
+24
View File
@@ -0,0 +1,24 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import Any
class SecPipeError(Exception):
"""Base exception for all SecPipe custom exceptions.
All domain exceptions should inherit from this base to enable
consistent exception handling and hierarchy navigation.
"""
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
"""Initialize SecPipe error.
:param message: Error message.
:param details: Optional error details dictionary.
"""
Exception.__init__(self, message)
self.message = message
self.details = details or {}
secpipe-common/src/secpipe_common/hub/__init__.py
+43
View File
@@ -0,0 +1,43 @@
"""SecPipe Hub - Generic MCP server bridge.
This module provides a generic bridge to connect SecPipe with any MCP server.
It allows AI agents to discover and execute tools from external MCP servers
(like mcp-security-hub) through the same interface as native SecPipe modules.
The hub is server-agnostic: it doesn't hardcode any specific tools or servers.
Instead, it dynamically discovers tools by connecting to configured MCP servers
and calling their `list_tools()` method.
Supported transport types:
- docker: Run MCP server as a Docker container with stdio transport
- command: Run MCP server as a local process with stdio transport
- sse: Connect to a remote MCP server via Server-Sent Events
"""
from secpipe_common.hub.client import HubClient, HubClientError, PersistentSession
from secpipe_common.hub.executor import HubExecutionResult, HubExecutor
from secpipe_common.hub.models import (
HubConfig,
HubServer,
HubServerConfig,
HubServerType,
HubTool,
HubToolParameter,
)
from secpipe_common.hub.registry import HubRegistry
__all__ = [
"HubClient",
"HubClientError",
"HubConfig",
"HubExecutionResult",
"HubExecutor",
"HubRegistry",
"HubServer",
"HubServerConfig",
"HubServerType",
"HubTool",
"HubToolParameter",
"PersistentSession",
]
secpipe-common/src/secpipe_common/hub/client.py
+753
View File
@@ -0,0 +1,753 @@
"""MCP client for communicating with hub servers.
This module provides a generic MCP client that can connect to any MCP server
via stdio (docker/command) or SSE transport. It handles:
- Starting containers/processes for stdio transport
- Connecting to SSE endpoints
- Discovering tools via list_tools()
- Executing tools via call_tool()
- Persistent container sessions for stateful interactions
"""
from __future__ import annotations
import asyncio
import json
import os
import subprocess
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, cast
from secpipe_common.hub.models import (
HubServer,
HubServerConfig,
HubServerType,
HubTool,
)
if TYPE_CHECKING:
from asyncio.subprocess import Process
from collections.abc import AsyncGenerator
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structlog logger instance.
:returns: Configured structlog logger.
"""
from structlog import get_logger # noqa: PLC0415
return cast("BoundLogger", get_logger())
class HubClientError(Exception):
"""Error in hub client operations."""
@dataclass
class PersistentSession:
"""A persistent container session with an active MCP connection.
Keeps a Docker container running between tool calls to allow
stateful interactions (e.g., radare2 analysis, long-running fuzzing).
"""
#: Server name this session belongs to.
server_name: str
#: Docker container name.
container_name: str
#: Underlying process (docker run).
process: Process
#: Stream reader (process stdout).
reader: asyncio.StreamReader
#: Stream writer (process stdin).
writer: asyncio.StreamWriter
#: Whether the MCP session has been initialized.
initialized: bool = False
#: Lock to serialise concurrent requests on the same session.
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
#: When the session was started.
started_at: datetime = field(default_factory=lambda: datetime.now(tz=timezone.utc))
#: Monotonic counter for JSON-RPC request IDs.
request_id: int = 0
@property
def alive(self) -> bool:
"""Check if the underlying process is still running."""
return self.process.returncode is None
class HubClient:
"""Client for communicating with MCP hub servers.
Supports stdio (via docker/command) and SSE transports.
Uses the MCP protocol for tool discovery and execution.
"""
#: Default timeout for operations.
DEFAULT_TIMEOUT: int = 30
def __init__(self, timeout: int = DEFAULT_TIMEOUT) -> None:
"""Initialize the hub client.
:param timeout: Default timeout for operations in seconds.
"""
self._timeout = timeout
self._persistent_sessions: dict[str, PersistentSession] = {}
self._request_id: int = 0
async def discover_tools(self, server: HubServer) -> list[HubTool]:
"""Discover tools from a hub server.
Connects to the server, calls list_tools(), and returns
parsed HubTool instances.
:param server: Hub server to discover tools from.
:returns: List of discovered tools.
:raises HubClientError: If discovery fails.
"""
logger = get_logger()
config = server.config
logger.info("Discovering tools", server=config.name, type=config.type.value)
try:
async with self._connect(config) as (reader, writer):
# Initialise MCP session (skip for persistent — already done)
if not self._persistent_sessions.get(config.name):
await self._initialize_session(reader, writer, config.name)
# List tools
tools_data = await self._call_method(
reader,
writer,
"tools/list",
{},
)
# Parse tools
tools = []
for tool_data in tools_data.get("tools", []):
tool = HubTool.from_mcp_tool(
server_name=config.name,
name=tool_data["name"],
description=tool_data.get("description"),
input_schema=tool_data.get("inputSchema", {}),
)
tools.append(tool)
logger.info(
"Discovered tools",
server=config.name,
count=len(tools),
)
return tools
except Exception as e:
logger.error(
"Tool discovery failed",
server=config.name,
error=str(e),
)
raise HubClientError(f"Discovery failed for {config.name}: {e}") from e
async def execute_tool(
self,
server: HubServer,
tool_name: str,
arguments: dict[str, Any],
*,
timeout: int | None = None,
extra_volumes: list[str] | None = None,
) -> dict[str, Any]:
"""Execute a tool on a hub server.
:param server: Hub server to execute on.
:param tool_name: Name of the tool to execute.
:param arguments: Tool arguments.
:param timeout: Execution timeout (uses default if None).
:param extra_volumes: Additional Docker volume mounts to inject.
:returns: Tool execution result.
:raises HubClientError: If execution fails.
"""
logger = get_logger()
config = server.config
exec_timeout = timeout or config.timeout or self._timeout
logger.info(
"Executing hub tool",
server=config.name,
tool=tool_name,
timeout=exec_timeout,
)
try:
async with self._connect(config, extra_volumes=extra_volumes) as (reader, writer):
# Initialise MCP session (skip for persistent — already done)
if not self._persistent_sessions.get(config.name):
await self._initialize_session(reader, writer, config.name)
# Call tool
result = await asyncio.wait_for(
self._call_method(
reader,
writer,
"tools/call",
{"name": tool_name, "arguments": arguments},
),
timeout=exec_timeout,
)
logger.info(
"Tool execution completed",
server=config.name,
tool=tool_name,
)
return result
except asyncio.TimeoutError as e:
logger.error(
"Tool execution timed out",
server=config.name,
tool=tool_name,
timeout=exec_timeout,
)
raise HubClientError(
f"Execution timed out for {config.name}:{tool_name}"
) from e
except Exception as e:
logger.error(
"Tool execution failed",
server=config.name,
tool=tool_name,
error=str(e),
)
raise HubClientError(
f"Execution failed for {config.name}:{tool_name}: {e}"
) from e
@asynccontextmanager
async def _connect(
self,
config: HubServerConfig,
extra_volumes: list[str] | None = None,
) -> AsyncGenerator[tuple[asyncio.StreamReader, asyncio.StreamWriter], None]:
"""Connect to an MCP server.
If a persistent session exists for this server, reuse it (with a lock
to serialise concurrent requests). Otherwise, fall through to the
ephemeral per-call connection logic.
:param config: Server configuration.
:param extra_volumes: Additional Docker volume mounts to inject.
:yields: Tuple of (reader, writer) for communication.
"""
# Check for active persistent session
session = self._persistent_sessions.get(config.name)
if session and session.initialized and session.alive:
async with session.lock:
yield session.reader, session.writer # type: ignore[misc]
return
# Ephemeral connection (original behaviour)
if config.type == HubServerType.DOCKER:
async with self._connect_docker(config, extra_volumes=extra_volumes) as streams:
yield streams
elif config.type == HubServerType.COMMAND:
async with self._connect_command(config) as streams:
yield streams
elif config.type == HubServerType.SSE:
async with self._connect_sse(config) as streams:
yield streams
else:
msg = f"Unsupported server type: {config.type}"
raise HubClientError(msg)
@asynccontextmanager
async def _connect_docker(
self,
config: HubServerConfig,
extra_volumes: list[str] | None = None,
) -> AsyncGenerator[tuple[asyncio.StreamReader, asyncio.StreamWriter], None]:
"""Connect to a Docker-based MCP server.
:param config: Server configuration with image name.
:param extra_volumes: Additional volume mounts to inject (e.g. project assets).
:yields: Tuple of (reader, writer) for stdio communication.
"""
if not config.image:
msg = f"Docker image not specified for server '{config.name}'"
raise HubClientError(msg)
# Build docker command
cmd = ["docker", "run", "-i", "--rm"]
# Add capabilities
for cap in config.capabilities:
cmd.extend(["--cap-add", cap])
# Add volumes from server config
for volume in config.volumes:
cmd.extend(["-v", os.path.expanduser(volume)])
# Add extra volumes (e.g. project assets injected at runtime)
for volume in (extra_volumes or []):
cmd.extend(["-v", os.path.expanduser(volume)])
# Add environment variables
for key, value in config.environment.items():
cmd.extend(["-e", f"{key}={value}"])
cmd.append(config.image)
# Use 4 MB buffer to handle large tool responses (YARA rulesets, trivy output, etc.)
_STREAM_LIMIT = 4 * 1024 * 1024
process: Process = await asyncio.create_subprocess_exec(
*cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
limit=_STREAM_LIMIT,
)
try:
if process.stdin is None or process.stdout is None:
msg = "Failed to get process streams"
raise HubClientError(msg)
# Create asyncio streams from process pipes
reader = process.stdout
writer = process.stdin
yield reader, writer # type: ignore[misc]
finally:
process.terminate()
try:
await asyncio.wait_for(process.wait(), timeout=5)
except asyncio.TimeoutError:
process.kill()
@asynccontextmanager
async def _connect_command(
self,
config: HubServerConfig,
) -> AsyncGenerator[tuple[asyncio.StreamReader, asyncio.StreamWriter], None]:
"""Connect to a command-based MCP server.
:param config: Server configuration with command.
:yields: Tuple of (reader, writer) for stdio communication.
"""
if not config.command:
msg = f"Command not specified for server '{config.name}'"
raise HubClientError(msg)
# Set up environment
env = dict(config.environment) if config.environment else None
# Use 4 MB buffer to handle large tool responses
_STREAM_LIMIT = 4 * 1024 * 1024
process: Process = await asyncio.create_subprocess_exec(
*config.command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
limit=_STREAM_LIMIT,
)
try:
if process.stdin is None or process.stdout is None:
msg = "Failed to get process streams"
raise HubClientError(msg)
reader = process.stdout
writer = process.stdin
yield reader, writer # type: ignore[misc]
finally:
process.terminate()
try:
await asyncio.wait_for(process.wait(), timeout=5)
except asyncio.TimeoutError:
process.kill()
@asynccontextmanager
async def _connect_sse(
self,
config: HubServerConfig,
) -> AsyncGenerator[tuple[asyncio.StreamReader, asyncio.StreamWriter], None]:
"""Connect to an SSE-based MCP server.
:param config: Server configuration with URL.
:yields: Tuple of (reader, writer) for SSE communication.
"""
# SSE support requires additional dependencies
# For now, raise not implemented
msg = "SSE transport not yet implemented"
raise NotImplementedError(msg)
async def _initialize_session(
self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
server_name: str,
) -> dict[str, Any]:
"""Initialize MCP session with the server.
:param reader: Stream reader.
:param writer: Stream writer.
:param server_name: Server name for logging.
:returns: Server capabilities.
"""
# Send initialize request
result = await self._call_method(
reader,
writer,
"initialize",
{
"protocolVersion": "2024-11-05",
"capabilities": {},
"clientInfo": {
"name": "secpipe-hub",
"version": "0.1.0",
},
},
)
# Send initialized notification
await self._send_notification(reader, writer, "notifications/initialized", {})
return result
async def _call_method(
self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
method: str,
params: dict[str, Any],
) -> dict[str, Any]:
"""Call an MCP method.
:param reader: Stream reader.
:param writer: Stream writer.
:param method: Method name.
:param params: Method parameters.
:returns: Method result.
"""
# Create JSON-RPC request with unique ID
self._request_id += 1
request = {
"jsonrpc": "2.0",
"id": self._request_id,
"method": method,
"params": params,
}
# Send request
request_line = json.dumps(request) + "\n"
writer.write(request_line.encode())
await writer.drain()
# Read response
response_line = await asyncio.wait_for(
reader.readline(),
timeout=self._timeout,
)
if not response_line:
msg = "Empty response from server"
raise HubClientError(msg)
response = json.loads(response_line.decode())
if "error" in response:
error = response["error"]
msg = f"MCP error: {error.get('message', 'Unknown error')}"
raise HubClientError(msg)
result = response.get("result", {})
# Check for tool-level errors in content items
for item in result.get("content", []):
if item.get("isError", False):
error_text = item.get("text", "unknown error")
msg = f"Tool returned error: {error_text}"
raise HubClientError(msg)
return result
async def _send_notification(
self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
method: str,
params: dict[str, Any],
) -> None:
"""Send an MCP notification (no response expected).
:param reader: Stream reader (unused but kept for consistency).
:param writer: Stream writer.
:param method: Notification method name.
:param params: Notification parameters.
"""
# Create JSON-RPC notification (no id)
notification = {
"jsonrpc": "2.0",
"method": method,
"params": params,
}
notification_line = json.dumps(notification) + "\n"
writer.write(notification_line.encode())
await writer.drain()
# ------------------------------------------------------------------
# Persistent session management
# ------------------------------------------------------------------
async def start_persistent_session(
self,
config: HubServerConfig,
extra_volumes: list[str] | None = None,
) -> PersistentSession:
"""Start a persistent Docker container and initialise MCP session.
The container stays running until :meth:`stop_persistent_session` is
called, allowing multiple tool calls on the same session.
:param config: Server configuration (must be Docker type).
:param extra_volumes: Additional host:container volume mounts to inject.
:returns: The created persistent session.
:raises HubClientError: If the container cannot be started.
"""
logger = get_logger()
if config.name in self._persistent_sessions:
session = self._persistent_sessions[config.name]
if session.alive:
logger.info("Persistent session already running", server=config.name)
return session
# Dead session — clean up and restart
await self._cleanup_session(config.name)
if config.type != HubServerType.DOCKER:
msg = f"Persistent mode only supports Docker servers (got {config.type.value})"
raise HubClientError(msg)
if not config.image:
msg = f"Docker image not specified for server '{config.name}'"
raise HubClientError(msg)
container_name = f"secpipe-{config.name}"
# Remove stale container with same name if it exists
try:
rm_proc = await asyncio.create_subprocess_exec(
"docker", "rm", "-f", container_name,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
await rm_proc.wait()
except Exception:
pass
# Build docker run command (no --rm, with --name)
cmd = ["docker", "run", "-i", "--name", container_name]
for cap in config.capabilities:
cmd.extend(["--cap-add", cap])
for volume in config.volumes:
cmd.extend(["-v", os.path.expanduser(volume)])
for extra_vol in (extra_volumes or []):
cmd.extend(["-v", extra_vol])
for key, value in config.environment.items():
cmd.extend(["-e", f"{key}={value}"])
cmd.append(config.image)
_STREAM_LIMIT = 4 * 1024 * 1024
logger.info(
"Starting persistent container",
server=config.name,
container=container_name,
image=config.image,
)
process: Process = await asyncio.create_subprocess_exec(
*cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
limit=_STREAM_LIMIT,
)
if process.stdin is None or process.stdout is None:
process.terminate()
msg = "Failed to get process streams"
raise HubClientError(msg)
session = PersistentSession(
server_name=config.name,
container_name=container_name,
process=process,
reader=process.stdout,
writer=process.stdin,
)
# Initialise MCP session
try:
await self._initialize_session(
session.reader, # type: ignore[arg-type]
session.writer, # type: ignore[arg-type]
config.name,
)
session.initialized = True
except Exception as e:
process.terminate()
try:
await asyncio.wait_for(process.wait(), timeout=5)
except asyncio.TimeoutError:
process.kill()
msg = f"Failed to initialise MCP session for {config.name}: {e}"
raise HubClientError(msg) from e
self._persistent_sessions[config.name] = session
logger.info(
"Persistent session started",
server=config.name,
container=container_name,
)
return session
async def stop_persistent_session(self, server_name: str) -> bool:
"""Stop a persistent container session.
:param server_name: Name of the server whose session to stop.
:returns: True if a session was stopped, False if none found.
"""
return await self._cleanup_session(server_name)
def get_persistent_session(self, server_name: str) -> PersistentSession | None:
"""Get a persistent session by server name.
:param server_name: Server name.
:returns: The session if running, None otherwise.
"""
session = self._persistent_sessions.get(server_name)
if session and not session.alive:
# Mark dead session — don't remove here to avoid async issues
return None
return session
def list_persistent_sessions(self) -> list[dict[str, Any]]:
"""List all persistent sessions with their status.
:returns: List of session info dictionaries.
"""
sessions = []
for name, session in self._persistent_sessions.items():
sessions.append({
"server_name": name,
"container_name": session.container_name,
"alive": session.alive,
"initialized": session.initialized,
"started_at": session.started_at.isoformat(),
"uptime_seconds": int(
(datetime.now(tz=timezone.utc) - session.started_at).total_seconds()
),
})
return sessions
async def stop_all_persistent_sessions(self) -> int:
"""Stop all persistent sessions.
:returns: Number of sessions stopped.
"""
names = list(self._persistent_sessions.keys())
count = 0
for name in names:
if await self._cleanup_session(name):
count += 1
return count
async def _cleanup_session(self, server_name: str) -> bool:
"""Clean up a persistent session (terminate process, remove container).
:param server_name: Server name.
:returns: True if cleaned up, False if not found.
"""
logger = get_logger()
session = self._persistent_sessions.pop(server_name, None)
if session is None:
return False
logger.info("Stopping persistent session", server=server_name)
# Terminate process
if session.alive:
session.process.terminate()
try:
await asyncio.wait_for(session.process.wait(), timeout=10)
except asyncio.TimeoutError:
session.process.kill()
await session.process.wait()
# Remove Docker container
try:
rm_proc = await asyncio.create_subprocess_exec(
"docker", "rm", "-f", session.container_name,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
await rm_proc.wait()
except Exception:
pass
logger.info(
"Persistent session stopped",
server=server_name,
container=session.container_name,
)
return True
secpipe-common/src/secpipe_common/hub/executor.py
+627
View File
@@ -0,0 +1,627 @@
"""Hub executor for managing MCP server lifecycle and tool execution.
This module provides a high-level interface for:
- Discovering tools from all registered hub servers
- Executing tools with proper error handling
- Managing the lifecycle of hub operations
"""
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from secpipe_common.hub.client import HubClient, HubClientError, PersistentSession
from secpipe_common.hub.models import HubServer, HubServerConfig, HubTool
from secpipe_common.hub.registry import HubRegistry
if TYPE_CHECKING:
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structlog logger instance.
:returns: Configured structlog logger.
"""
from structlog import get_logger # noqa: PLC0415
return cast("BoundLogger", get_logger())
class HubExecutionResult:
"""Result of a hub tool execution."""
def __init__(
self,
*,
success: bool,
server_name: str,
tool_name: str,
result: dict[str, Any] | None = None,
error: str | None = None,
) -> None:
"""Initialize execution result.
:param success: Whether execution succeeded.
:param server_name: Name of the hub server.
:param tool_name: Name of the executed tool.
:param result: Tool execution result data.
:param error: Error message if execution failed.
"""
self.success = success
self.server_name = server_name
self.tool_name = tool_name
self.result = result or {}
self.error = error
@property
def identifier(self) -> str:
"""Get full tool identifier."""
return f"hub:{self.server_name}:{self.tool_name}"
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary.
:returns: Dictionary representation.
"""
return {
"success": self.success,
"identifier": self.identifier,
"server": self.server_name,
"tool": self.tool_name,
"result": self.result,
"error": self.error,
}
class HubExecutor:
"""Executor for hub server operations.
Provides high-level methods for discovering and executing
tools from hub servers.
"""
#: Hub registry instance.
_registry: HubRegistry
#: MCP client instance.
_client: HubClient
def __init__(
self,
config_path: Path | None = None,
timeout: int = 300,
) -> None:
"""Initialize the hub executor.
:param config_path: Path to hub-servers.json config file.
:param timeout: Default timeout for tool execution.
"""
self._registry = HubRegistry(config_path)
self._client = HubClient(timeout=timeout)
self._continuous_sessions: dict[str, dict[str, Any]] = {}
@property
def registry(self) -> HubRegistry:
"""Get the hub registry.
:returns: Hub registry instance.
"""
return self._registry
def add_server(self, config: HubServerConfig) -> HubServer:
"""Add a server to the registry.
:param config: Server configuration.
:returns: Created HubServer instance.
"""
return self._registry.add_server(config)
async def discover_all_tools(self) -> dict[str, list[HubTool]]:
"""Discover tools from all enabled servers.
:returns: Dict mapping server names to lists of discovered tools.
"""
logger = get_logger()
results: dict[str, list[HubTool]] = {}
for server in self._registry.enabled_servers:
try:
tools = await self._client.discover_tools(server)
self._registry.update_server_tools(server.name, tools)
results[server.name] = tools
except HubClientError as e:
logger.warning(
"Failed to discover tools",
server=server.name,
error=str(e),
)
self._registry.update_server_tools(server.name, [], error=str(e))
results[server.name] = []
return results
async def discover_server_tools(self, server_name: str) -> list[HubTool]:
"""Discover tools from a specific server.
:param server_name: Name of the server.
:returns: List of discovered tools.
:raises ValueError: If server not found.
"""
server = self._registry.get_server(server_name)
if not server:
msg = f"Server '{server_name}' not found"
raise ValueError(msg)
try:
tools = await self._client.discover_tools(server)
self._registry.update_server_tools(server_name, tools)
return tools
except HubClientError as e:
self._registry.update_server_tools(server_name, [], error=str(e))
raise
async def execute_tool(
self,
identifier: str,
arguments: dict[str, Any] | None = None,
*,
timeout: int | None = None,
extra_volumes: list[str] | None = None,
) -> HubExecutionResult:
"""Execute a hub tool.
:param identifier: Tool identifier (hub:server:tool or server:tool).
:param arguments: Tool arguments.
:param timeout: Execution timeout.
:param extra_volumes: Additional Docker volume mounts to inject.
:returns: Execution result.
"""
logger = get_logger()
arguments = arguments or {}
# Parse identifier and find tool
server, tool = self._registry.find_tool(identifier)
if not server or not tool:
# Try to parse as server:tool and discover
parts = identifier.replace("hub:", "").split(":")
if len(parts) == 2: # noqa: PLR2004
server_name, tool_name = parts
server = self._registry.get_server(server_name)
if server and not server.discovered:
# Try to discover tools first
try:
await self.discover_server_tools(server_name)
tool = server.get_tool(tool_name)
except HubClientError:
pass
if server and not tool:
# Tool not found, but server exists - try to execute anyway
# The server might have the tool even if discovery failed
tool_name_to_use = tool_name
else:
tool_name_to_use = tool.name if tool else ""
if not server:
return HubExecutionResult(
success=False,
server_name=server_name,
tool_name=tool_name,
error=f"Server '{server_name}' not found",
)
# Execute even if tool wasn't discovered (server might still have it)
try:
result = await self._client.execute_tool(
server,
tool_name_to_use or tool_name,
arguments,
timeout=timeout,
extra_volumes=extra_volumes,
)
return HubExecutionResult(
success=True,
server_name=server.name,
tool_name=tool_name_to_use or tool_name,
result=result,
)
except HubClientError as e:
return HubExecutionResult(
success=False,
server_name=server.name,
tool_name=tool_name_to_use or tool_name,
error=str(e),
)
else:
return HubExecutionResult(
success=False,
server_name="unknown",
tool_name=identifier,
error=f"Invalid tool identifier: {identifier}",
)
# Execute the tool
logger.info(
"Executing hub tool",
server=server.name,
tool=tool.name,
arguments=arguments,
)
try:
result = await self._client.execute_tool(
server,
tool.name,
arguments,
timeout=timeout,
extra_volumes=extra_volumes,
)
return HubExecutionResult(
success=True,
server_name=server.name,
tool_name=tool.name,
result=result,
)
except HubClientError as e:
return HubExecutionResult(
success=False,
server_name=server.name,
tool_name=tool.name,
error=str(e),
)
def list_servers(self) -> list[dict[str, Any]]:
"""List all registered servers with their status.
:returns: List of server info dicts.
"""
servers = []
for server in self._registry.servers:
session = self._client.get_persistent_session(server.name)
servers.append({
"name": server.name,
"identifier": server.identifier,
"type": server.config.type.value,
"enabled": server.config.enabled,
"category": server.config.category,
"description": server.config.description,
"persistent": server.config.persistent,
"persistent_session_active": session is not None and session.alive,
"discovered": server.discovered,
"tool_count": len(server.tools),
"error": server.discovery_error,
})
return servers
def list_tools(self) -> list[dict[str, Any]]:
"""List all discovered tools.
:returns: List of tool info dicts.
"""
tools = []
for tool in self._registry.get_all_tools():
tools.append({
"identifier": tool.identifier,
"name": tool.name,
"server": tool.server_name,
"description": tool.description,
"parameters": [p.model_dump() for p in tool.parameters],
})
return tools
def get_tool_schema(self, identifier: str) -> dict[str, Any] | None:
"""Get the JSON Schema for a tool's input.
:param identifier: Tool identifier.
:returns: JSON Schema dict or None if not found.
"""
_, tool = self._registry.find_tool(identifier)
if tool:
return tool.input_schema
return None
# ------------------------------------------------------------------
# Persistent session management
# ------------------------------------------------------------------
async def start_persistent_server(self, server_name: str, extra_volumes: list[str] | None = None) -> dict[str, Any]:
"""Start a persistent container session for a server.
The container stays running between tool calls, allowing stateful
interactions (e.g., radare2 sessions, long-running fuzzing).
:param server_name: Name of the hub server to start.
:param extra_volumes: Additional host:container volume mounts to inject.
:returns: Session status dictionary.
:raises ValueError: If server not found.
"""
logger = get_logger()
server = self._registry.get_server(server_name)
if not server:
msg = f"Server '{server_name}' not found"
raise ValueError(msg)
session = await self._client.start_persistent_session(server.config, extra_volumes=extra_volumes)
# Auto-discover tools on the new session
try:
tools = await self._client.discover_tools(server)
self._registry.update_server_tools(server_name, tools)
except HubClientError as e:
logger.warning(
"Tool discovery failed on persistent session",
server=server_name,
error=str(e),
)
# Include discovered tools in the result so agent knows what's available
discovered_tools = []
server_obj = self._registry.get_server(server_name)
if server_obj:
for tool in server_obj.tools:
discovered_tools.append({
"identifier": tool.identifier,
"name": tool.name,
"description": tool.description,
})
return {
"server_name": session.server_name,
"container_name": session.container_name,
"alive": session.alive,
"initialized": session.initialized,
"started_at": session.started_at.isoformat(),
"tools": discovered_tools,
"tool_count": len(discovered_tools),
}
async def stop_persistent_server(self, server_name: str) -> bool:
"""Stop a persistent container session.
:param server_name: Server name.
:returns: True if a session was stopped.
"""
return await self._client.stop_persistent_session(server_name)
def get_persistent_status(self, server_name: str) -> dict[str, Any] | None:
"""Get status of a persistent session.
:param server_name: Server name.
:returns: Status dict or None if no session.
"""
session = self._client.get_persistent_session(server_name)
if not session:
return None
from datetime import datetime, timezone # noqa: PLC0415
return {
"server_name": session.server_name,
"container_name": session.container_name,
"alive": session.alive,
"initialized": session.initialized,
"started_at": session.started_at.isoformat(),
"uptime_seconds": int(
(datetime.now(tz=timezone.utc) - session.started_at).total_seconds()
),
}
def list_persistent_sessions(self) -> list[dict[str, Any]]:
"""List all persistent sessions.
:returns: List of session status dicts.
"""
return self._client.list_persistent_sessions()
async def stop_all_persistent_servers(self) -> int:
"""Stop all persistent sessions.
:returns: Number of sessions stopped.
"""
return await self._client.stop_all_persistent_sessions()
# ------------------------------------------------------------------
# Continuous session management
# ------------------------------------------------------------------
async def start_continuous_tool(
self,
server_name: str,
start_tool: str,
arguments: dict[str, Any],
) -> dict[str, Any]:
"""Start a continuous hub tool session.
Ensures a persistent container is running, then calls the start tool
(e.g., ``cargo_fuzz_start``) which returns a session_id. Tracks the
session for subsequent status/stop calls.
:param server_name: Hub server name.
:param start_tool: Name of the start tool on the server.
:param arguments: Arguments for the start tool.
:returns: Start result including session_id.
:raises ValueError: If server not found.
"""
logger = get_logger()
server = self._registry.get_server(server_name)
if not server:
msg = f"Server '{server_name}' not found"
raise ValueError(msg)
# Ensure persistent session is running
persistent = self._client.get_persistent_session(server_name)
if not persistent or not persistent.alive:
logger.info(
"Auto-starting persistent session for continuous tool",
server=server_name,
)
await self._client.start_persistent_session(server.config)
# Discover tools on the new session
try:
tools = await self._client.discover_tools(server)
self._registry.update_server_tools(server_name, tools)
except HubClientError as e:
logger.warning(
"Tool discovery failed on persistent session",
server=server_name,
error=str(e),
)
# Call the start tool
result = await self._client.execute_tool(
server, start_tool, arguments,
)
# Extract session_id from result
content_text = ""
for item in result.get("content", []):
if item.get("type") == "text":
content_text = item.get("text", "")
break
import json # noqa: PLC0415
try:
start_result = json.loads(content_text) if content_text else result
except json.JSONDecodeError:
start_result = result
session_id = start_result.get("session_id", "")
if session_id:
from datetime import datetime, timezone # noqa: PLC0415
self._continuous_sessions[session_id] = {
"session_id": session_id,
"server_name": server_name,
"start_tool": start_tool,
"status_tool": start_tool.replace("_start", "_status"),
"stop_tool": start_tool.replace("_start", "_stop"),
"started_at": datetime.now(tz=timezone.utc).isoformat(),
"status": "running",
}
return start_result
async def get_continuous_tool_status(
self,
session_id: str,
) -> dict[str, Any]:
"""Get status of a continuous hub tool session.
:param session_id: Session ID from start_continuous_tool.
:returns: Status dict from the hub server's status tool.
:raises ValueError: If session not found.
"""
session_info = self._continuous_sessions.get(session_id)
if not session_info:
msg = f"Unknown continuous session: {session_id}"
raise ValueError(msg)
server = self._registry.get_server(session_info["server_name"])
if not server:
msg = f"Server '{session_info['server_name']}' not found"
raise ValueError(msg)
result = await self._client.execute_tool(
server,
session_info["status_tool"],
{"session_id": session_id},
)
# Parse the text content
content_text = ""
for item in result.get("content", []):
if item.get("type") == "text":
content_text = item.get("text", "")
break
import json # noqa: PLC0415
try:
return json.loads(content_text) if content_text else result
except json.JSONDecodeError:
return result
async def stop_continuous_tool(
self,
session_id: str,
) -> dict[str, Any]:
"""Stop a continuous hub tool session.
:param session_id: Session ID to stop.
:returns: Final results from the hub server's stop tool.
:raises ValueError: If session not found.
"""
session_info = self._continuous_sessions.get(session_id)
if not session_info:
msg = f"Unknown continuous session: {session_id}"
raise ValueError(msg)
server = self._registry.get_server(session_info["server_name"])
if not server:
msg = f"Server '{session_info['server_name']}' not found"
raise ValueError(msg)
result = await self._client.execute_tool(
server,
session_info["stop_tool"],
{"session_id": session_id},
)
# Parse the text content
content_text = ""
for item in result.get("content", []):
if item.get("type") == "text":
content_text = item.get("text", "")
break
import json # noqa: PLC0415
try:
stop_result = json.loads(content_text) if content_text else result
except json.JSONDecodeError:
stop_result = result
# Update session tracking
session_info["status"] = "stopped"
return stop_result
def list_continuous_sessions(self) -> list[dict[str, Any]]:
"""List all tracked continuous sessions.
:returns: List of continuous session info dicts.
"""
return list(self._continuous_sessions.values())
secpipe-common/src/secpipe_common/hub/models.py
+310
View File
@@ -0,0 +1,310 @@
"""Data models for SecPipe Hub.
This module defines the Pydantic models used to represent MCP servers
and their tools in the hub registry.
"""
from __future__ import annotations
from enum import Enum
from typing import Any
from pydantic import BaseModel, Field
class HubServerType(str, Enum):
"""Type of MCP server connection."""
#: Run as Docker container with stdio transport.
DOCKER = "docker"
#: Run as local command/process with stdio transport.
COMMAND = "command"
#: Connect via Server-Sent Events (HTTP).
SSE = "sse"
class HubServerConfig(BaseModel):
"""Configuration for an MCP server in the hub.
This defines how to connect to an MCP server, not what tools it provides.
Tools are discovered dynamically at runtime.
"""
#: Unique identifier for this server (e.g., "nmap", "nuclei").
name: str = Field(description="Unique server identifier")
#: Human-readable description of the server.
description: str | None = Field(
default=None,
description="Human-readable description",
)
#: Type of connection to use.
type: HubServerType = Field(description="Connection type")
#: Docker image name (for type=docker).
image: str | None = Field(
default=None,
description="Docker image name (for docker type)",
)
#: Command to run (for type=command).
command: list[str] | None = Field(
default=None,
description="Command and args (for command type)",
)
#: URL endpoint (for type=sse).
url: str | None = Field(
default=None,
description="SSE endpoint URL (for sse type)",
)
#: Environment variables to pass to the server.
environment: dict[str, str] = Field(
default_factory=dict,
description="Environment variables",
)
#: Docker capabilities to add (e.g., ["NET_RAW"] for nmap).
capabilities: list[str] = Field(
default_factory=list,
description="Docker capabilities to add",
)
#: Volume mounts for Docker (e.g., ["/host/path:/container/path:ro"]).
volumes: list[str] = Field(
default_factory=list,
description="Docker volume mounts",
)
#: Whether this server is enabled.
enabled: bool = Field(
default=True,
description="Whether server is enabled",
)
#: Category for grouping (e.g., "reconnaissance", "web-security").
category: str | None = Field(
default=None,
description="Category for grouping servers",
)
#: Per-server timeout override in seconds (None = use default_timeout).
timeout: int | None = Field(
default=None,
description="Per-server execution timeout override in seconds",
)
#: Whether to use persistent container mode (keep container running between calls).
persistent: bool = Field(
default=False,
description="Keep container running between tool calls for stateful interactions",
)
class HubToolParameter(BaseModel):
"""A parameter for an MCP tool.
Parsed from the tool's JSON Schema inputSchema.
"""
#: Parameter name.
name: str
#: Parameter type (string, integer, boolean, array, object).
type: str
#: Human-readable description.
description: str | None = None
#: Whether this parameter is required.
required: bool = False
#: Default value if any.
default: Any = None
#: Enum values if constrained.
enum: list[Any] | None = None
class HubTool(BaseModel):
"""An MCP tool discovered from a hub server.
This is populated by calling `list_tools()` on the MCP server.
"""
#: Tool name as defined by the MCP server.
name: str = Field(description="Tool name from MCP server")
#: Human-readable description.
description: str | None = Field(
default=None,
description="Tool description",
)
#: Name of the hub server this tool belongs to.
server_name: str = Field(description="Parent server name")
#: Parsed parameters from inputSchema.
parameters: list[HubToolParameter] = Field(
default_factory=list,
description="Tool parameters",
)
#: Raw JSON Schema for the tool input.
input_schema: dict[str, Any] = Field(
default_factory=dict,
description="Raw JSON Schema from MCP",
)
@property
def identifier(self) -> str:
"""Get the full tool identifier (hub:server:tool)."""
return f"hub:{self.server_name}:{self.name}"
@classmethod
def from_mcp_tool(
cls,
server_name: str,
name: str,
description: str | None,
input_schema: dict[str, Any],
) -> HubTool:
"""Create a HubTool from MCP tool metadata.
:param server_name: Name of the parent hub server.
:param name: Tool name.
:param description: Tool description.
:param input_schema: JSON Schema for tool input.
:returns: HubTool instance.
"""
parameters = cls._parse_parameters(input_schema)
return cls(
name=name,
description=description,
server_name=server_name,
parameters=parameters,
input_schema=input_schema,
)
@staticmethod
def _parse_parameters(schema: dict[str, Any]) -> list[HubToolParameter]:
"""Parse parameters from JSON Schema.
:param schema: JSON Schema dict.
:returns: List of parsed parameters.
"""
parameters: list[HubToolParameter] = []
properties = schema.get("properties", {})
required_params = set(schema.get("required", []))
for name, prop in properties.items():
param = HubToolParameter(
name=name,
type=prop.get("type", "string"),
description=prop.get("description"),
required=name in required_params,
default=prop.get("default"),
enum=prop.get("enum"),
)
parameters.append(param)
return parameters
class HubServer(BaseModel):
"""A hub server with its discovered tools.
Combines configuration with dynamically discovered tools.
"""
#: Server configuration.
config: HubServerConfig
#: Tools discovered from the server (populated at runtime).
tools: list[HubTool] = Field(
default_factory=list,
description="Discovered tools",
)
#: Whether tools have been discovered.
discovered: bool = Field(
default=False,
description="Whether tools have been discovered",
)
#: Error message if discovery failed.
discovery_error: str | None = Field(
default=None,
description="Error message if discovery failed",
)
@property
def name(self) -> str:
"""Get server name."""
return self.config.name
@property
def identifier(self) -> str:
"""Get server identifier for module listing."""
return f"hub:{self.config.name}"
def get_tool(self, tool_name: str) -> HubTool | None:
"""Get a tool by name.
:param tool_name: Name of the tool.
:returns: HubTool if found, None otherwise.
"""
for tool in self.tools:
if tool.name == tool_name:
return tool
return None
class HubConfig(BaseModel):
"""Configuration for the entire hub.
Loaded from hub-servers.json or similar config file.
"""
#: List of configured servers.
servers: list[HubServerConfig] = Field(
default_factory=list,
description="Configured MCP servers",
)
#: Default timeout for tool execution (seconds).
default_timeout: int = Field(
default=300,
description="Default execution timeout",
)
#: Whether to cache discovered tools.
cache_tools: bool = Field(
default=True,
description="Cache discovered tools",
)
#: Workflow hints indexed by "after:<tool_name>" keys.
#: Loaded inline or merged from workflow_hints_file.
workflow_hints: dict[str, Any] = Field(
default_factory=dict,
description="Workflow hints indexed by 'after:<tool_name>'",
)
#: Optional path to an external workflow-hints.json file.
#: Relative paths are resolved relative to the hub-config.json location.
workflow_hints_file: str | None = Field(
default=None,
description="Path to an external workflow-hints.json to load and merge",
)
secpipe-common/src/secpipe_common/hub/registry.py
+289
View File
@@ -0,0 +1,289 @@
"""Hub registry for managing MCP server configurations.
The registry loads server configurations from a JSON file and provides
methods to access and manage them. It does not hardcode any specific
servers or tools - everything is configured by the user.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import TYPE_CHECKING, cast
from secpipe_common.hub.models import (
HubConfig,
HubServer,
HubServerConfig,
)
if TYPE_CHECKING:
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structlog logger instance.
:returns: Configured structlog logger.
"""
from structlog import get_logger # noqa: PLC0415
return cast("BoundLogger", get_logger())
class HubRegistry:
"""Registry for MCP hub servers.
Manages the configuration and state of hub servers.
Configurations are loaded from a JSON file.
"""
#: Loaded hub configuration.
_config: HubConfig
#: Server instances with discovered tools.
_servers: dict[str, HubServer]
#: Path to the configuration file.
_config_path: Path | None
def __init__(self, config_path: Path | str | None = None) -> None:
"""Initialize the hub registry.
:param config_path: Path to hub-servers.json config file.
If None, starts with empty configuration.
"""
if config_path is not None:
self._config_path = Path(config_path)
else:
self._config_path = None
self._servers = {}
self._config = HubConfig()
if self._config_path and self._config_path.exists():
self._load_config(self._config_path)
def _load_config(self, config_path: Path) -> None:
"""Load configuration from JSON file.
:param config_path: Path to config file.
"""
logger = get_logger()
try:
with config_path.open() as f:
data = json.load(f)
self._config = HubConfig.model_validate(data)
# Create server instances from config
for server_config in self._config.servers:
if server_config.enabled:
self._servers[server_config.name] = HubServer(
config=server_config,
)
# Load and merge external workflow hints file if specified.
if self._config.workflow_hints_file:
hints_path = Path(self._config.workflow_hints_file)
if not hints_path.is_absolute():
hints_path = config_path.parent / hints_path
if hints_path.exists():
try:
with hints_path.open() as hf:
hints_data = json.load(hf)
self._config.workflow_hints.update(hints_data.get("hints", {}))
logger.info(
"Loaded workflow hints",
path=str(hints_path),
hints=len(self._config.workflow_hints),
)
except Exception as hints_err:
logger.warning(
"Failed to load workflow hints file",
path=str(hints_path),
error=str(hints_err),
)
logger.info(
"Loaded hub configuration",
path=str(config_path),
servers=len(self._servers),
)
except Exception as e:
logger.error(
"Failed to load hub configuration",
path=str(config_path),
error=str(e),
)
raise
def reload(self) -> None:
"""Reload configuration from file."""
if self._config_path and self._config_path.exists():
self._servers.clear()
self._load_config(self._config_path)
@property
def servers(self) -> list[HubServer]:
"""Get all registered servers.
:returns: List of hub servers.
"""
return list(self._servers.values())
@property
def enabled_servers(self) -> list[HubServer]:
"""Get all enabled servers.
:returns: List of enabled hub servers.
"""
return [s for s in self._servers.values() if s.config.enabled]
def get_server(self, name: str) -> HubServer | None:
"""Get a server by name.
:param name: Server name.
:returns: HubServer if found, None otherwise.
"""
return self._servers.get(name)
def add_server(self, config: HubServerConfig) -> HubServer:
"""Add a server to the registry.
:param config: Server configuration.
:returns: Created HubServer instance.
:raises ValueError: If server with same name exists.
"""
if config.name in self._servers:
msg = f"Server '{config.name}' already exists"
raise ValueError(msg)
server = HubServer(config=config)
self._servers[config.name] = server
self._config.servers.append(config)
get_logger().info("Added hub server", name=config.name, type=config.type)
return server
def remove_server(self, name: str) -> bool:
"""Remove a server from the registry.
:param name: Server name.
:returns: True if removed, False if not found.
"""
if name not in self._servers:
return False
del self._servers[name]
self._config.servers = [s for s in self._config.servers if s.name != name]
get_logger().info("Removed hub server", name=name)
return True
def save_config(self, path: Path | None = None) -> None:
"""Save current configuration to file.
:param path: Path to save to. Uses original path if None.
"""
save_path = path or self._config_path
if not save_path:
msg = "No config path specified"
raise ValueError(msg)
with save_path.open("w") as f:
json.dump(
self._config.model_dump(mode="json"),
f,
indent=2,
)
get_logger().info("Saved hub configuration", path=str(save_path))
def update_server_tools(
self,
server_name: str,
tools: list,
*,
error: str | None = None,
) -> None:
"""Update discovered tools for a server.
Called by the hub client after tool discovery.
:param server_name: Server name.
:param tools: List of HubTool instances.
:param error: Error message if discovery failed.
"""
server = self._servers.get(server_name)
if not server:
return
if error:
server.discovered = False
server.discovery_error = error
server.tools = []
else:
server.discovered = True
server.discovery_error = None
server.tools = tools
def get_workflow_hint(self, tool_name: str) -> dict | None:
"""Get the workflow hint for a tool by name.
:param tool_name: Tool name (e.g. ``binwalk_extract``).
:returns: Hint dict for the ``after:<tool_name>`` key, or None.
"""
return self._config.workflow_hints.get(f"after:{tool_name}") or None
def get_all_tools(self) -> list:
"""Get all discovered tools from all servers.
:returns: Flat list of all HubTool instances.
"""
tools = []
for server in self._servers.values():
if server.discovered:
tools.extend(server.tools)
return tools
def find_tool(self, identifier: str):
"""Find a tool by its full identifier.
:param identifier: Full identifier (hub:server:tool or server:tool).
:returns: Tuple of (HubServer, HubTool) if found, (None, None) otherwise.
"""
# Parse identifier
parts = identifier.split(":")
if len(parts) == 3 and parts[0] == "hub": # noqa: PLR2004
# hub:server:tool format
server_name = parts[1]
tool_name = parts[2]
elif len(parts) == 2: # noqa: PLR2004
# server:tool format
server_name = parts[0]
tool_name = parts[1]
else:
return None, None
server = self._servers.get(server_name)
if not server:
return None, None
tool = server.get_tool(tool_name)
return server, tool
secpipe-common/src/secpipe_common/py.typed
View File
secpipe-common/src/secpipe_common/sandboxes/__init__.py
+23
View File
@@ -0,0 +1,23 @@
"""SecPipe sandbox abstractions and implementations."""
from secpipe_common.sandboxes.engines import (
AbstractSecPipeEngineConfiguration,
AbstractSecPipeSandboxEngine,
Docker,
DockerConfiguration,
SecPipeSandboxEngines,
ImageInfo,
Podman,
PodmanConfiguration,
)
__all__ = [
"AbstractSecPipeEngineConfiguration",
"AbstractSecPipeSandboxEngine",
"Docker",
"DockerConfiguration",
"SecPipeSandboxEngines",
"ImageInfo",
"Podman",
"PodmanConfiguration",
]
secpipe-common/src/secpipe_common/sandboxes/engines/__init__.py
+21
View File
@@ -0,0 +1,21 @@
"""Container engine implementations for SecPipe sandboxes."""
from secpipe_common.sandboxes.engines.base import (
AbstractSecPipeEngineConfiguration,
AbstractSecPipeSandboxEngine,
ImageInfo,
)
from secpipe_common.sandboxes.engines.docker import Docker, DockerConfiguration
from secpipe_common.sandboxes.engines.enumeration import SecPipeSandboxEngines
from secpipe_common.sandboxes.engines.podman import Podman, PodmanConfiguration
__all__ = [
"AbstractSecPipeEngineConfiguration",
"AbstractSecPipeSandboxEngine",
"Docker",
"DockerConfiguration",
"SecPipeSandboxEngines",
"ImageInfo",
"Podman",
"PodmanConfiguration",
]
secpipe-common/src/secpipe_common/sandboxes/engines/base/__init__.py
+15
View File
@@ -0,0 +1,15 @@
"""Base engine abstractions."""
from secpipe_common.sandboxes.engines.base.configuration import (
AbstractSecPipeEngineConfiguration,
)
from secpipe_common.sandboxes.engines.base.engine import (
AbstractSecPipeSandboxEngine,
ImageInfo,
)
__all__ = [
"AbstractSecPipeEngineConfiguration",
"AbstractSecPipeSandboxEngine",
"ImageInfo",
]
secpipe-common/src/secpipe_common/sandboxes/engines/base/configuration.py
+24
View File
@@ -0,0 +1,24 @@
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from pydantic import BaseModel
from secpipe_common.sandboxes.engines.enumeration import (
SecPipeSandboxEngines,
)
if TYPE_CHECKING:
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine
class AbstractSecPipeEngineConfiguration(ABC, BaseModel):
"""TODO."""
#: TODO.
kind: SecPipeSandboxEngines
@abstractmethod
def into_engine(self) -> AbstractSecPipeSandboxEngine:
"""TODO."""
message: str = f"method 'into_engine' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
secpipe-common/src/secpipe_common/sandboxes/engines/base/engine.py
+315
View File
@@ -0,0 +1,315 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from pathlib import Path, PurePath
@dataclass
class ImageInfo:
"""Information about a container image."""
#: Full image reference (e.g., "localhost/secpipe-module-echidna:latest").
reference: str
#: Repository name (e.g., "localhost/secpipe-module-echidna").
repository: str
#: Image tag (e.g., "latest").
tag: str
#: Image ID (short hash).
image_id: str | None = None
#: Image size in bytes.
size: int | None = None
#: Image labels/metadata.
labels: dict[str, str] | None = None
class AbstractSecPipeSandboxEngine(ABC):
"""Abstract class used as a base for all SecPipe sandbox engine classes."""
@abstractmethod
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
:returns: List of ImageInfo objects for available images.
"""
message: str = f"method 'list_images' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def register_archive(self, archive: Path, repository: str) -> None:
"""TODO.
:param archive: TODO.
"""
message: str = f"method 'register_archive' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox based on the given image.
:param image: The image the sandbox should be based on.
:returns: The sandbox identifier.
"""
message: str = f"method 'spawn_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:param destination: TODO.
"""
message: str = f"method 'push_archive_to_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def start_sandbox(self, identifier: str) -> None:
"""TODO.
:param identifier: The identifier of the sandbox to start.
"""
message: str = f"method 'start_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
:param sandbox: The identifier of the sandbox.
:param command: The command to run.
"""
message: str = f"method 'execute_inside_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:returns: TODO.
"""
message: str = f"method 'pull_archive_from_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to terminate.
"""
message: str = f"method 'terminate_sandbox' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
# -------------------------------------------------------------------------
# Extended Container Operations
# -------------------------------------------------------------------------
@abstractmethod
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference (e.g., "localhost/module:latest").
:returns: True if image exists, False otherwise.
"""
message: str = f"method 'image_exists' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference to pull.
:param timeout: Timeout in seconds for the pull operation.
:raises SecPipeError: If pull fails.
"""
message: str = f"method 'pull_image' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
message: str = f"method 'tag_image' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
message: str = f"method 'create_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for it to complete.
:param identifier: Container identifier.
:param timeout: Timeout in seconds for execution.
:returns: Tuple of (exit_code, stdout, stderr).
"""
message: str = f"method 'start_container_attached' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
message: str = f"method 'copy_to_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
message: str = f"method 'copy_from_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal even if running.
"""
message: str = f"method 'remove_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
# -------------------------------------------------------------------------
# Continuous/Background Execution Operations
# -------------------------------------------------------------------------
@abstractmethod
def start_container(self, identifier: str) -> None:
"""Start a container without waiting for it to complete (detached mode).
:param identifier: Container identifier.
"""
message: str = f"method 'start_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status (e.g., "running", "exited", "created").
"""
message: str = f"method 'get_container_status' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container gracefully.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
message: str = f"method 'stop_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a running container using exec.
:param identifier: Container identifier.
:param path: Path to file inside container.
:returns: File contents as string.
"""
message: str = f"method 'read_file_from_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def tail_file_from_container(self, identifier: str, path: str, start_line: int = 1) -> str:
"""Read a file from a running container starting at a given line number.
Uses ``tail -n +{start_line}`` to avoid re-reading the entire file on
every poll. This is the preferred method for incremental reads of
append-only files such as ``stream.jsonl``.
:param identifier: Container identifier.
:param path: Path to file inside container.
:param start_line: 1-based line number to start reading from.
:returns: File contents from *start_line* onwards (may be empty).
"""
message: str = f"method 'tail_file_from_container' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
message: str = f"method 'list_containers' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
@abstractmethod
def read_file_from_image(self, image: str, path: str) -> str:
"""Read a file from inside an image without starting a container.
Creates a temporary container, copies the file, and removes the container.
:param image: Image reference (e.g., "secpipe-rust-analyzer:latest").
:param path: Path to file inside image.
:returns: File contents as string.
"""
message: str = f"method 'read_file_from_image' is not implemented for class '{self.__class__.__name__}'"
raise NotImplementedError(message)
secpipe-common/src/secpipe_common/sandboxes/engines/docker/__init__.py
+13
View File
@@ -0,0 +1,13 @@
"""Docker container engine implementation."""
from secpipe_common.sandboxes.engines.docker.cli import DockerCLI
from secpipe_common.sandboxes.engines.docker.configuration import (
DockerConfiguration,
)
from secpipe_common.sandboxes.engines.docker.engine import Docker
__all__ = [
"Docker",
"DockerCLI",
"DockerConfiguration",
]
secpipe-common/src/secpipe_common/sandboxes/engines/docker/cli.py
+460
View File
@@ -0,0 +1,460 @@
"""Docker CLI engine.
This engine uses subprocess calls to the Docker CLI, providing a simple
and portable interface that works on Linux, macOS, and Windows wherever
Docker Desktop or Docker Engine is installed.
"""
from __future__ import annotations
import json
import subprocess
from pathlib import Path, PurePath
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, cast
from secpipe_common.exceptions import SecPipeError
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structured logger."""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
class DockerCLI(AbstractSecPipeSandboxEngine):
"""Docker engine using CLI commands.
This implementation uses subprocess calls to the Docker CLI,
providing a simple and portable interface that works wherever
Docker is installed (Docker Desktop on macOS/Windows, Docker Engine on Linux).
"""
def __init__(self) -> None:
"""Initialize the DockerCLI engine."""
AbstractSecPipeSandboxEngine.__init__(self)
def _base_cmd(self) -> list[str]:
"""Get base Docker command.
:returns: Base command list.
"""
return ["docker"]
def _run(self, args: list[str], *, check: bool = True, capture: bool = True) -> subprocess.CompletedProcess:
"""Run a Docker command.
:param args: Command arguments (without 'docker').
:param check: Raise exception on non-zero exit.
:param capture: Capture stdout/stderr.
:returns: CompletedProcess result.
"""
cmd = self._base_cmd() + args
get_logger().debug("running docker command", cmd=" ".join(cmd))
return subprocess.run(
cmd,
check=check,
capture_output=capture,
text=True,
)
# -------------------------------------------------------------------------
# Image Operations
# -------------------------------------------------------------------------
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images.
:returns: List of ImageInfo objects.
"""
result = self._run(["images", "--format", "json"])
images: list[ImageInfo] = []
try:
# Docker outputs one JSON object per line
lines = result.stdout.strip().split("\n") if result.stdout.strip() else []
data = [json.loads(line) for line in lines if line.strip()]
except json.JSONDecodeError:
get_logger().warning("failed to parse docker images output")
return images
for image in data:
repo = image.get("Repository", "")
tag = image.get("Tag", "latest")
if repo == "<none>":
continue
reference = f"{repo}:{tag}"
if filter_prefix and filter_prefix not in reference:
continue
# Try to get labels from image inspect
labels = {}
try:
inspect_result = self._run(["image", "inspect", reference], check=False)
if inspect_result.returncode == 0:
inspect_data = json.loads(inspect_result.stdout)
if inspect_data and len(inspect_data) > 0:
labels = inspect_data[0].get("Config", {}).get("Labels") or {}
except (json.JSONDecodeError, IndexError):
pass
images.append(
ImageInfo(
reference=reference,
repository=repo,
tag=tag,
image_id=image.get("ID", "")[:12],
size=image.get("Size"),
labels=labels,
)
)
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
return images
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference.
:returns: True if image exists.
"""
result = self._run(["image", "inspect", image], check=False)
return result.returncode == 0
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference.
:param timeout: Timeout in seconds.
"""
get_logger().info("pulling image", image=image)
try:
self._run(["pull", image])
get_logger().info("image pulled successfully", image=image)
except subprocess.CalledProcessError as exc:
message = f"Failed to pull image '{image}': {exc.stderr}"
raise SecPipeError(message) from exc
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
self._run(["tag", source, target])
get_logger().debug("tagged image", source=source, target=target)
def build_image(self, context_path: Path, tag: str, dockerfile: str = "Dockerfile") -> None:
"""Build an image from a Dockerfile.
:param context_path: Path to build context.
:param tag: Image tag.
:param dockerfile: Dockerfile name.
"""
get_logger().info("building image", tag=tag, context=str(context_path))
self._run(["build", "-t", tag, "-f", dockerfile, str(context_path)])
get_logger().info("image built successfully", tag=tag)
def register_archive(self, archive: Path, repository: str) -> None:
"""Load an image from a tar archive.
:param archive: Path to tar archive.
:param repository: Repository name for the loaded image.
"""
result = self._run(["load", "-i", str(archive)])
# Tag the loaded image
for line in result.stdout.splitlines():
if "Loaded image:" in line:
loaded_image = line.split("Loaded image:")[-1].strip()
self._run(["tag", loaded_image, f"{repository}:latest"])
break
get_logger().debug("registered archive", archive=str(archive), repository=repository)
# -------------------------------------------------------------------------
# Container Operations
# -------------------------------------------------------------------------
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox (container) from an image.
:param image: Image to create container from.
:returns: Container identifier.
"""
result = self._run(["create", image])
container_id = result.stdout.strip()
get_logger().debug("created container", container_id=container_id)
return container_id
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
args = ["create"]
if volumes:
for host_path, container_path in volumes.items():
args.extend(["-v", f"{host_path}:{container_path}:ro"])
args.append(image)
result = self._run(args)
container_id = result.stdout.strip()
get_logger().debug("created container", container_id=container_id, image=image)
return container_id
def start_sandbox(self, identifier: str) -> None:
"""Start a container.
:param identifier: Container identifier.
"""
self._run(["start", identifier])
get_logger().debug("started container", container_id=identifier)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting.
:param identifier: Container identifier.
"""
self._run(["start", identifier])
get_logger().debug("started container (detached)", container_id=identifier)
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for completion.
:param identifier: Container identifier.
:param timeout: Timeout in seconds.
:returns: Tuple of (exit_code, stdout, stderr).
"""
get_logger().debug("starting container attached", container_id=identifier)
# Start the container
self._run(["start", identifier])
# Wait for completion
wait_result = self._run(["wait", identifier])
exit_code = int(wait_result.stdout.strip()) if wait_result.stdout.strip() else -1
# Get logs
stdout_result = self._run(["logs", identifier], check=False)
stdout_str = stdout_result.stdout or ""
stderr_str = stdout_result.stderr or ""
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
return (exit_code, stdout_str, stderr_str)
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside a container.
:param identifier: Container identifier.
:param command: Command to run.
"""
get_logger().debug("executing command in container", container_id=identifier)
self._run(["exec", identifier] + command)
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""Copy an archive to a container.
:param identifier: Container identifier.
:param source: Source archive path.
:param destination: Destination path in container.
"""
get_logger().debug("copying to container", container_id=identifier, source=str(source))
self._run(["cp", str(source), f"{identifier}:{destination}"])
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""Copy files from a container to a local archive.
:param identifier: Container identifier.
:param source: Source path in container.
:returns: Path to local archive.
"""
get_logger().debug("copying from container", container_id=identifier, source=str(source))
with NamedTemporaryFile(delete=False, delete_on_close=False, suffix=".tar") as tmp:
self._run(["cp", f"{identifier}:{source}", tmp.name])
return Path(tmp.name)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
self._run(["cp", str(source), f"{identifier}:{destination}"])
get_logger().debug("copied to container", source=str(source), destination=destination)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
destination.mkdir(parents=True, exist_ok=True)
self._run(["cp", f"{identifier}:{source}", str(destination)])
get_logger().debug("copied from container", source=source, destination=str(destination))
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate and remove a container.
:param identifier: Container identifier.
"""
# Stop if running
self._run(["stop", identifier], check=False)
# Remove
self._run(["rm", "-f", identifier], check=False)
get_logger().debug("terminated container", container_id=identifier)
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal.
"""
args = ["rm"]
if force:
args.append("-f")
args.append(identifier)
self._run(args, check=False)
get_logger().debug("removed container", container_id=identifier)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
self._run(["stop", "-t", str(timeout), identifier], check=False)
get_logger().debug("stopped container", container_id=identifier)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status.
"""
result = self._run(["inspect", "--format", "{{.State.Status}}", identifier], check=False)
return result.stdout.strip() if result.returncode == 0 else "unknown"
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a container.
:param identifier: Container identifier.
:param path: Path to file in container.
:returns: File contents.
"""
result = self._run(["exec", identifier, "cat", path], check=False)
if result.returncode != 0:
get_logger().debug("failed to read file from container", path=path)
return ""
return result.stdout
def tail_file_from_container(self, identifier: str, path: str, start_line: int = 1) -> str:
"""Read a file from a container starting at a given line number.
:param identifier: Container identifier.
:param path: Path to file in container.
:param start_line: 1-based line number to start reading from.
:returns: File contents from *start_line* onwards.
"""
result = self._run(
["exec", identifier, "tail", "-n", f"+{start_line}", path],
check=False,
)
if result.returncode != 0:
get_logger().debug("failed to tail file from container", path=path, start_line=start_line)
return ""
return result.stdout
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
args = ["ps", "--format", "json"]
if all_containers:
args.append("-a")
result = self._run(args)
try:
# Docker outputs one JSON object per line
lines = result.stdout.strip().split("\n") if result.stdout.strip() else []
data = [json.loads(line) for line in lines if line.strip()]
return [
{
"Id": c.get("ID", ""),
"Names": c.get("Names", ""),
"Status": c.get("State", ""),
"Image": c.get("Image", ""),
}
for c in data
]
except json.JSONDecodeError:
return []
def read_file_from_image(self, image: str, path: str) -> str:
"""Read a file from inside an image without starting a long-running container.
Uses docker run with --entrypoint override to read the file via cat.
:param image: Image reference (e.g., "secpipe-rust-analyzer:latest").
:param path: Path to file inside image.
:returns: File contents as string.
"""
logger = get_logger()
# Use docker run with --entrypoint to override any container entrypoint
result = self._run(
["run", "--rm", "--entrypoint", "cat", image, path],
check=False,
)
if result.returncode != 0:
logger.debug("failed to read file from image", image=image, path=path, stderr=result.stderr)
return ""
return result.stdout
secpipe-common/src/secpipe_common/sandboxes/engines/docker/configuration.py
+22
View File
@@ -0,0 +1,22 @@
from typing import TYPE_CHECKING, Literal
from secpipe_common.sandboxes.engines.base.configuration import AbstractSecPipeEngineConfiguration
from secpipe_common.sandboxes.engines.docker.engine import Docker
from secpipe_common.sandboxes.engines.enumeration import SecPipeSandboxEngines
if TYPE_CHECKING:
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine
class DockerConfiguration(AbstractSecPipeEngineConfiguration):
"""TODO."""
#: TODO.
kind: Literal[SecPipeSandboxEngines.DOCKER] = SecPipeSandboxEngines.DOCKER
#: TODO.
socket: str
def into_engine(self) -> AbstractSecPipeSandboxEngine:
"""TODO."""
return Docker(socket=self.socket)
secpipe-common/src/secpipe_common/sandboxes/engines/docker/engine.py
+184
View File
@@ -0,0 +1,184 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from pathlib import Path, PurePath
class Docker(AbstractSecPipeSandboxEngine):
"""TODO."""
#: TODO.
__socket: str
def __init__(self, socket: str) -> None:
"""Initialize an instance of the class.
:param socket: TODO.
"""
super().__init__()
self.__socket = socket
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
:returns: List of ImageInfo objects for available images.
"""
# TODO: Implement Docker image listing
message: str = "Docker engine list_images is not yet implemented"
raise NotImplementedError(message)
def register_archive(self, archive: Path, repository: str) -> None:
"""TODO.
:param archive: TODO.
"""
return super().register_archive(archive=archive, repository=repository)
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox based on the given image.
:param image: The image the sandbox should be based on.
:returns: The sandbox identifier.
"""
return super().spawn_sandbox(image)
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:param destination: TODO.
"""
super().push_archive_to_sandbox(identifier, source, destination)
def start_sandbox(self, identifier: str) -> None:
"""TODO.
:param identifier: The identifier of the sandbox to start.
"""
super().start_sandbox(identifier)
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
:param sandbox: The identifier of the sandbox.
:param command: The command to run.
"""
super().execute_inside_sandbox(identifier, command)
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:returns: TODO.
"""
return super().pull_archive_from_sandbox(identifier, source)
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to terminate.
"""
super().terminate_sandbox(identifier)
# -------------------------------------------------------------------------
# Extended Container Operations (stubs - not yet implemented)
# -------------------------------------------------------------------------
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally."""
message: str = "Docker engine image_exists is not yet implemented"
raise NotImplementedError(message)
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry."""
message: str = "Docker engine pull_image is not yet implemented"
raise NotImplementedError(message)
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name."""
message: str = "Docker engine tag_image is not yet implemented"
raise NotImplementedError(message)
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image."""
message: str = "Docker engine create_container is not yet implemented"
raise NotImplementedError(message)
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for it to complete."""
message: str = "Docker engine start_container_attached is not yet implemented"
raise NotImplementedError(message)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container."""
message: str = "Docker engine copy_to_container is not yet implemented"
raise NotImplementedError(message)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container."""
message: str = "Docker engine copy_from_container is not yet implemented"
raise NotImplementedError(message)
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container."""
message: str = "Docker engine remove_container is not yet implemented"
raise NotImplementedError(message)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting for it to complete."""
message: str = "Docker engine start_container is not yet implemented"
raise NotImplementedError(message)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container."""
message: str = "Docker engine get_container_status is not yet implemented"
raise NotImplementedError(message)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container gracefully."""
message: str = "Docker engine stop_container is not yet implemented"
raise NotImplementedError(message)
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a running container using exec."""
message: str = "Docker engine read_file_from_container is not yet implemented"
raise NotImplementedError(message)
def tail_file_from_container(self, identifier: str, path: str, start_line: int = 1) -> str:
"""Read a file from a container starting at a given line number."""
message: str = "Docker engine tail_file_from_container is not yet implemented"
raise NotImplementedError(message)
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers."""
message: str = "Docker engine list_containers is not yet implemented"
raise NotImplementedError(message)
def read_file_from_image(self, image: str, path: str) -> str:
"""Read a file from inside an image without starting a long-running container."""
message: str = "Docker engine read_file_from_image is not yet implemented"
raise NotImplementedError(message)
secpipe-common/src/secpipe_common/sandboxes/engines/enumeration.py
+11
View File
@@ -0,0 +1,11 @@
from enum import StrEnum
class SecPipeSandboxEngines(StrEnum):
"""TODO."""
#: TODO.
DOCKER = "docker"
#: TODO.
PODMAN = "podman"
secpipe-common/src/secpipe_common/sandboxes/engines/podman/__init__.py
+13
View File
@@ -0,0 +1,13 @@
"""Podman container engine implementation."""
from secpipe_common.sandboxes.engines.podman.cli import PodmanCLI
from secpipe_common.sandboxes.engines.podman.configuration import (
PodmanConfiguration,
)
from secpipe_common.sandboxes.engines.podman.engine import Podman
__all__ = [
"Podman",
"PodmanCLI",
"PodmanConfiguration",
]
secpipe-common/src/secpipe_common/sandboxes/engines/podman/cli.py
+536
View File
@@ -0,0 +1,536 @@
"""Podman CLI engine with custom storage support.
This engine uses subprocess calls to the Podman CLI instead of the socket API,
allowing for custom storage paths (--root, --runroot) that work regardless of
system Podman configuration or snap environment issues.
"""
from __future__ import annotations
import json
import subprocess
import tarfile
from io import BytesIO
from pathlib import Path, PurePath
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, cast
from secpipe_common.exceptions import SecPipeError
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""Get structured logger."""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
def _is_running_under_snap() -> bool:
"""Check if running under Snap environment.
VS Code installed via Snap sets XDG_DATA_HOME to a version-specific path,
causing Podman to look for storage in non-standard locations. When SNAP
is set, we use custom storage paths to ensure consistency.
Note: Snap only exists on Linux, so this also handles macOS implicitly.
"""
import os # noqa: PLC0415
return os.getenv("SNAP") is not None
class PodmanCLI(AbstractSecPipeSandboxEngine):
"""Podman engine using CLI with custom storage paths.
This implementation uses subprocess calls to the Podman CLI with --root
and --runroot flags when running under Snap, providing isolation from
system Podman storage.
The custom storage is only used when:
1. Running under Snap (SNAP env var is set) - to fix XDG_DATA_HOME issues
2. Custom paths are explicitly provided
Otherwise, uses default Podman storage which works for:
- Native Linux installations
- macOS (where Podman runs in a VM via podman machine)
"""
__graphroot: Path | None
__runroot: Path | None
__use_custom_storage: bool
def __init__(self, graphroot: Path | None = None, runroot: Path | None = None) -> None:
"""Initialize the PodmanCLI engine.
:param graphroot: Path to container image storage.
:param runroot: Path to container runtime state.
Custom storage is used when running under Snap AND paths are provided.
:raises SecPipeError: If running on macOS (Podman not supported).
"""
import sys # noqa: PLC0415
if sys.platform == "darwin":
msg = (
"Podman is not supported on macOS. Please use Docker instead:\n"
" brew install --cask docker\n"
" # Or download from https://docker.com/products/docker-desktop"
)
raise SecPipeError(msg)
AbstractSecPipeSandboxEngine.__init__(self)
# Use custom storage only under Snap (to fix XDG_DATA_HOME issues)
self.__use_custom_storage = _is_running_under_snap() and graphroot is not None and runroot is not None
if self.__use_custom_storage:
self.__graphroot = graphroot
self.__runroot = runroot
# Ensure directories exist
self.__graphroot.mkdir(parents=True, exist_ok=True)
self.__runroot.mkdir(parents=True, exist_ok=True)
else:
self.__graphroot = None
self.__runroot = None
def _base_cmd(self) -> list[str]:
"""Get base Podman command with storage flags.
:returns: Base command list, with --root and --runroot only under Snap.
"""
if self.__use_custom_storage and self.__graphroot and self.__runroot:
return [
"podman",
"--root",
str(self.__graphroot),
"--runroot",
str(self.__runroot),
]
return ["podman"]
def _run(self, args: list[str], *, check: bool = True, capture: bool = True) -> subprocess.CompletedProcess:
"""Run a Podman command.
:param args: Command arguments (without 'podman').
:param check: Raise exception on non-zero exit.
:param capture: Capture stdout/stderr.
:returns: CompletedProcess result.
"""
cmd = self._base_cmd() + args
get_logger().debug("running podman command", cmd=" ".join(cmd))
return subprocess.run(
cmd,
check=check,
capture_output=capture,
text=True,
)
# -------------------------------------------------------------------------
# Image Operations
# -------------------------------------------------------------------------
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images.
:returns: List of ImageInfo objects.
"""
result = self._run(["images", "--format", "json"])
images: list[ImageInfo] = []
try:
data = json.loads(result.stdout) if result.stdout.strip() else []
except json.JSONDecodeError:
get_logger().warning("failed to parse podman images output")
return images
for image in data:
# Get repository and tag from Names
names = image.get("Names") or []
for name in names:
if filter_prefix and not name.startswith(filter_prefix):
continue
# Parse repository and tag
if ":" in name:
repo, tag = name.rsplit(":", 1)
else:
repo = name
tag = "latest"
# Get labels if available
labels = image.get("Labels") or {}
images.append(
ImageInfo(
reference=name,
repository=repo,
tag=tag,
image_id=image.get("Id", "")[:12],
size=image.get("Size"),
labels=labels,
)
)
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
return images
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference.
:returns: True if image exists.
"""
result = self._run(["image", "exists", image], check=False)
return result.returncode == 0
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference.
:param timeout: Timeout in seconds.
"""
get_logger().info("pulling image", image=image)
try:
self._run(["pull", image])
get_logger().info("image pulled successfully", image=image)
except subprocess.CalledProcessError as exc:
message = f"Failed to pull image '{image}': {exc.stderr}"
raise SecPipeError(message) from exc
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
self._run(["tag", source, target])
get_logger().debug("tagged image", source=source, target=target)
def build_image(self, context_path: Path, tag: str, dockerfile: str = "Dockerfile") -> None:
"""Build an image from a Dockerfile.
:param context_path: Path to build context.
:param tag: Image tag.
:param dockerfile: Dockerfile name.
"""
get_logger().info("building image", tag=tag, context=str(context_path))
self._run(["build", "-t", tag, "-f", dockerfile, str(context_path)])
get_logger().info("image built successfully", tag=tag)
def register_archive(self, archive: Path, repository: str) -> None:
"""Load an image from a tar archive.
:param archive: Path to tar archive.
:param repository: Repository name for the loaded image.
"""
result = self._run(["load", "-i", str(archive)])
# Tag the loaded image
# Parse loaded image ID from output
for line in result.stdout.splitlines():
if "Loaded image:" in line:
loaded_image = line.split("Loaded image:")[-1].strip()
self._run(["tag", loaded_image, f"{repository}:latest"])
break
get_logger().debug("registered archive", archive=str(archive), repository=repository)
# -------------------------------------------------------------------------
# Container Operations
# -------------------------------------------------------------------------
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox (container) from an image.
:param image: Image to create container from.
:returns: Container identifier.
"""
result = self._run(["create", image])
container_id = result.stdout.strip()
get_logger().debug("created container", container_id=container_id)
return container_id
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
args = ["create"]
if volumes:
for host_path, container_path in volumes.items():
args.extend(["-v", f"{host_path}:{container_path}:ro"])
args.append(image)
result = self._run(args)
container_id = result.stdout.strip()
get_logger().debug("created container", container_id=container_id, image=image)
return container_id
def start_sandbox(self, identifier: str) -> None:
"""Start a container.
:param identifier: Container identifier.
"""
self._run(["start", identifier])
get_logger().debug("started container", container_id=identifier)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting.
:param identifier: Container identifier.
"""
self._run(["start", identifier])
get_logger().debug("started container (detached)", container_id=identifier)
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for completion.
:param identifier: Container identifier.
:param timeout: Timeout in seconds.
:returns: Tuple of (exit_code, stdout, stderr).
"""
get_logger().debug("starting container attached", container_id=identifier)
# Start the container
self._run(["start", identifier])
# Wait for completion
wait_result = self._run(["wait", identifier])
exit_code = int(wait_result.stdout.strip()) if wait_result.stdout.strip() else -1
# Get logs
stdout_result = self._run(["logs", identifier], check=False)
stdout_str = stdout_result.stdout or ""
stderr_str = stdout_result.stderr or ""
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
return (exit_code, stdout_str, stderr_str)
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside a container.
:param identifier: Container identifier.
:param command: Command to run.
"""
get_logger().debug("executing command in container", container_id=identifier)
self._run(["exec", identifier] + command)
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""Copy an archive to a container.
:param identifier: Container identifier.
:param source: Source archive path.
:param destination: Destination path in container.
"""
get_logger().debug("copying to container", container_id=identifier, source=str(source))
self._run(["cp", str(source), f"{identifier}:{destination}"])
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""Copy files from a container to a local archive.
:param identifier: Container identifier.
:param source: Source path in container.
:returns: Path to local archive.
"""
get_logger().debug("copying from container", container_id=identifier, source=str(source))
with NamedTemporaryFile(delete=False, delete_on_close=False, suffix=".tar") as tmp:
self._run(["cp", f"{identifier}:{source}", tmp.name])
return Path(tmp.name)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
self._run(["cp", str(source), f"{identifier}:{destination}"])
get_logger().debug("copied to container", source=str(source), destination=destination)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
destination.mkdir(parents=True, exist_ok=True)
self._run(["cp", f"{identifier}:{source}", str(destination)])
get_logger().debug("copied from container", source=source, destination=str(destination))
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate and remove a container.
:param identifier: Container identifier.
"""
# Stop if running
self._run(["stop", identifier], check=False)
# Remove
self._run(["rm", "-f", identifier], check=False)
get_logger().debug("terminated container", container_id=identifier)
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal.
"""
args = ["rm"]
if force:
args.append("-f")
args.append(identifier)
self._run(args, check=False)
get_logger().debug("removed container", container_id=identifier)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
self._run(["stop", "-t", str(timeout), identifier], check=False)
get_logger().debug("stopped container", container_id=identifier)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status.
"""
result = self._run(["inspect", "--format", "{{.State.Status}}", identifier], check=False)
return result.stdout.strip() if result.returncode == 0 else "unknown"
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a container.
:param identifier: Container identifier.
:param path: Path to file in container.
:returns: File contents.
"""
result = self._run(["exec", identifier, "cat", path], check=False)
if result.returncode != 0:
get_logger().debug("failed to read file from container", path=path)
return ""
return result.stdout
def tail_file_from_container(self, identifier: str, path: str, start_line: int = 1) -> str:
"""Read a file from a container starting at a given line number.
:param identifier: Container identifier.
:param path: Path to file in container.
:param start_line: 1-based line number to start reading from.
:returns: File contents from *start_line* onwards.
"""
result = self._run(
["exec", identifier, "tail", "-n", f"+{start_line}", path],
check=False,
)
if result.returncode != 0:
get_logger().debug("failed to tail file from container", path=path, start_line=start_line)
return ""
return result.stdout
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
args = ["ps", "--format", "json"]
if all_containers:
args.append("-a")
result = self._run(args)
try:
data = json.loads(result.stdout) if result.stdout.strip() else []
# Handle both list and single object responses
if isinstance(data, dict):
data = [data]
return [
{
"Id": c.get("Id", ""),
"Names": c.get("Names", []),
"Status": c.get("State", ""),
"Image": c.get("Image", ""),
}
for c in data
]
except json.JSONDecodeError:
return []
def read_file_from_image(self, image: str, path: str) -> str:
"""Read a file from inside an image without starting a long-running container.
Uses podman run with --entrypoint override to read the file via cat.
:param image: Image reference (e.g., "secpipe-rust-analyzer:latest").
:param path: Path to file inside image.
:returns: File contents as string.
"""
logger = get_logger()
# Use podman run with --entrypoint to override any container entrypoint
result = self._run(
["run", "--rm", "--entrypoint", "cat", image, path],
check=False,
)
if result.returncode != 0:
logger.debug("failed to read file from image", image=image, path=path, stderr=result.stderr)
return ""
return result.stdout
# -------------------------------------------------------------------------
# Utility Methods
# -------------------------------------------------------------------------
def get_storage_info(self) -> dict:
"""Get storage configuration info.
:returns: Dict with graphroot and runroot paths.
"""
return {
"graphroot": str(self.__graphroot),
"runroot": str(self.__runroot),
}
secpipe-common/src/secpipe_common/sandboxes/engines/podman/configuration.py
+22
View File
@@ -0,0 +1,22 @@
from typing import TYPE_CHECKING, Literal
from secpipe_common.sandboxes.engines.base.configuration import AbstractSecPipeEngineConfiguration
from secpipe_common.sandboxes.engines.enumeration import SecPipeSandboxEngines
from secpipe_common.sandboxes.engines.podman.engine import Podman
if TYPE_CHECKING:
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine
class PodmanConfiguration(AbstractSecPipeEngineConfiguration):
"""TODO."""
#: TODO.
kind: Literal[SecPipeSandboxEngines.PODMAN] = SecPipeSandboxEngines.PODMAN
#: TODO.
socket: str
def into_engine(self) -> AbstractSecPipeSandboxEngine:
"""TODO."""
return Podman(socket=self.socket)
secpipe-common/src/secpipe_common/sandboxes/engines/podman/engine.py
+557
View File
@@ -0,0 +1,557 @@
from __future__ import annotations
import tarfile
from io import BytesIO
from pathlib import Path, PurePath
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, cast
from podman.errors import ImageNotFound
from secpipe_common.exceptions import SecPipeError
from secpipe_common.sandboxes.engines.base.engine import AbstractSecPipeSandboxEngine, ImageInfo
if TYPE_CHECKING:
from podman import PodmanClient
from podman.domain.containers import Container
from structlog.stdlib import BoundLogger
def get_logger() -> BoundLogger:
"""TODO."""
from structlog import get_logger # noqa: PLC0415 (required by temporal)
return cast("BoundLogger", get_logger())
class Podman(AbstractSecPipeSandboxEngine):
"""TODO."""
#: TODO.
__socket: str
def __init__(self, socket: str) -> None:
"""Initialize an instance of the class.
:param socket: TODO.
"""
AbstractSecPipeSandboxEngine.__init__(self)
self.__socket = socket
def get_client(self) -> PodmanClient:
"""TODO.
:returns TODO.
"""
from podman import PodmanClient # noqa: PLC0415 (required by temporal)
return PodmanClient(base_url=self.__socket)
def list_images(self, filter_prefix: str | None = None) -> list[ImageInfo]:
"""List available container images.
:param filter_prefix: Optional prefix to filter images (e.g., "localhost/").
:returns: List of ImageInfo objects for available images.
"""
client: PodmanClient = self.get_client()
images: list[ImageInfo] = []
with client:
for image in client.images.list():
# Get all tags for this image
tags = image.tags or []
for tag in tags:
# Apply filter if specified
if filter_prefix and not tag.startswith(filter_prefix):
continue
# Parse repository and tag
if ":" in tag:
repo, tag_name = tag.rsplit(":", 1)
else:
repo = tag
tag_name = "latest"
images.append(
ImageInfo(
reference=tag,
repository=repo,
tag=tag_name,
image_id=image.short_id if hasattr(image, "short_id") else image.id[:12],
size=image.attrs.get("Size") if hasattr(image, "attrs") else None,
)
)
get_logger().debug("listed images", count=len(images), filter_prefix=filter_prefix)
return images
def register_archive(self, archive: Path, repository: str) -> None:
"""TODO.
:param archive: TODO.
"""
client: PodmanClient = self.get_client()
with client:
images = list(client.images.load(file_path=archive))
if len(images) != 1:
message: str = "expected only one image"
raise SecPipeError(message)
image = images[0]
image.tag(repository=repository, tag="latest")
def spawn_sandbox(self, image: str) -> str:
"""Spawn a sandbox based on the given image.
:param image: The image the sandbox should be based on.
:returns: The sandbox identifier.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.create(image=image)
container_identifier: str = container.id
get_logger().debug("create podman container", container_identifier=container_identifier)
return container_identifier
def push_archive_to_sandbox(self, identifier: str, source: Path, destination: PurePath) -> None:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:param destination: TODO.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"push archive to podman container",
container_identifier=identifier,
container_status=container.status,
)
# reading everything at once for now, even though this temporary solution is not viable with large files,
# since the podman sdk does not currently expose a way to chunk uploads.
# in order to fix this issue, we could directly interact with the podman rest api or make a contribution
# to the podman sdk in order to allow the 'put_archive' method to support chunked uploads.
data: bytes = source.read_bytes()
container.put_archive(path=str(destination), data=data)
def start_sandbox(self, identifier: str) -> None:
"""Start the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to start.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"start podman container",
container_identifier=identifier,
container_status=container.status,
)
container.start()
def execute_inside_sandbox(self, identifier: str, command: list[str]) -> None:
"""Execute a command inside the sandbox matching the given identifier and wait for completion.
:param sandbox: The identifier of the sandbox.
:param command: The command to run.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"executing command inside podman container",
container_identifier=identifier,
container_status=container.status,
)
(status, (stdout, stderr)) = container.exec_run(cmd=command, demux=True)
get_logger().debug(
"command execution result",
status=status,
stdout_size=len(stdout) if stdout else 0,
stderr_size=len(stderr) if stderr else 0,
)
def pull_archive_from_sandbox(self, identifier: str, source: PurePath) -> Path:
"""TODO.
:param identifier: TODO.
:param source: TODO.
:returns: TODO.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"pull archive from podman container",
container_identifier=identifier,
container_status=container.status,
)
with NamedTemporaryFile(delete=False, delete_on_close=False) as file:
stream, _stat = container.get_archive(path=str(source))
for chunk in stream:
file.write(chunk)
get_logger().debug(
"created archive",
archive=file.name,
)
return Path(file.name)
def terminate_sandbox(self, identifier: str) -> None:
"""Terminate the sandbox matching the given identifier.
:param identifier: The identifier of the sandbox to terminate.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug(
"kill podman container",
container_identifier=identifier,
container_status=container.status,
)
# Only kill running containers; for created/stopped, skip to remove
if container.status in ("running", "paused"):
container.kill()
get_logger().debug(
"remove podman container",
container_identifier=identifier,
container_status=container.status,
)
container.remove()
# -------------------------------------------------------------------------
# Extended Container Operations
# -------------------------------------------------------------------------
def image_exists(self, image: str) -> bool:
"""Check if a container image exists locally.
:param image: Full image reference (e.g., "localhost/module:latest").
:returns: True if image exists, False otherwise.
"""
client: PodmanClient = self.get_client()
with client:
try:
client.images.get(name=image)
except ImageNotFound:
return False
else:
return True
def pull_image(self, image: str, timeout: int = 300) -> None:
"""Pull an image from a container registry.
:param image: Full image reference to pull.
:param timeout: Timeout in seconds for the pull operation.
:raises SecPipeError: If pull fails.
"""
client: PodmanClient = self.get_client()
with client:
try:
get_logger().info("pulling image", image=image)
client.images.pull(repository=image)
get_logger().info("image pulled successfully", image=image)
except Exception as exc:
message = f"Failed to pull image '{image}': {exc}"
raise SecPipeError(message) from exc
def tag_image(self, source: str, target: str) -> None:
"""Tag an image with a new name.
:param source: Source image reference.
:param target: Target image reference.
"""
client: PodmanClient = self.get_client()
with client:
image = client.images.get(name=source)
# Parse target into repository and tag
if ":" in target:
repo, tag = target.rsplit(":", 1)
else:
repo = target
tag = "latest"
image.tag(repository=repo, tag=tag)
get_logger().debug("tagged image", source=source, target=target)
def create_container(
self,
image: str,
volumes: dict[str, str] | None = None,
) -> str:
"""Create a container from an image.
:param image: Image to create container from.
:param volumes: Optional volume mappings {host_path: container_path}.
:returns: Container identifier.
"""
client: PodmanClient = self.get_client()
with client:
# Build volume mounts in podman format
mounts = []
if volumes:
for host_path, container_path in volumes.items():
mounts.append({"type": "bind", "source": host_path, "target": container_path, "read_only": True})
container: Container = client.containers.create(image=image, mounts=mounts if mounts else None)
container_id: str = str(container.id)
get_logger().debug("created container", container_id=container_id, image=image)
return container_id
def start_container_attached(
self,
identifier: str,
timeout: int = 600,
) -> tuple[int, str, str]:
"""Start a container and wait for it to complete.
:param identifier: Container identifier.
:param timeout: Timeout in seconds for execution.
:returns: Tuple of (exit_code, stdout, stderr).
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
get_logger().debug("starting container attached", container_id=identifier)
# Start the container
container.start()
# Wait for completion with timeout
result = container.wait(timeout=timeout)
exit_code: int = result.get("StatusCode", -1) if isinstance(result, dict) else int(result)
# Get logs
stdout_raw = container.logs(stdout=True, stderr=False)
stderr_raw = container.logs(stdout=False, stderr=True)
# Decode if bytes
stdout_str: str = ""
stderr_str: str = ""
if isinstance(stdout_raw, bytes):
stdout_str = stdout_raw.decode("utf-8", errors="replace")
elif isinstance(stdout_raw, str):
stdout_str = stdout_raw
if isinstance(stderr_raw, bytes):
stderr_str = stderr_raw.decode("utf-8", errors="replace")
elif isinstance(stderr_raw, str):
stderr_str = stderr_raw
get_logger().debug("container finished", container_id=identifier, exit_code=exit_code)
return (exit_code, stdout_str, stderr_str)
def copy_to_container(self, identifier: str, source: Path, destination: str) -> None:
"""Copy a file or directory to a container.
:param identifier: Container identifier.
:param source: Source path on host.
:param destination: Destination path in container.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
# Create tar archive in memory
tar_buffer = BytesIO()
with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
tar.add(str(source), arcname=Path(source).name)
tar_buffer.seek(0)
# Use put_archive to copy
container.put_archive(path=destination, data=tar_buffer.read())
get_logger().debug("copied to container", source=str(source), destination=destination)
def copy_from_container(self, identifier: str, source: str, destination: Path) -> None:
"""Copy a file or directory from a container.
:param identifier: Container identifier.
:param source: Source path in container.
:param destination: Destination path on host.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
# Get archive from container
stream, _stat = container.get_archive(path=source)
# Write to temp file and extract
tar_buffer = BytesIO()
for chunk in stream:
tar_buffer.write(chunk)
tar_buffer.seek(0)
# Extract to destination
destination.mkdir(parents=True, exist_ok=True)
with tarfile.open(fileobj=tar_buffer, mode="r") as tar:
tar.extractall(path=destination) # noqa: S202 (trusted source)
get_logger().debug("copied from container", source=source, destination=str(destination))
def remove_container(self, identifier: str, *, force: bool = False) -> None:
"""Remove a container.
:param identifier: Container identifier.
:param force: Force removal even if running.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
if force and container.status in ("running", "paused"):
container.kill()
container.remove()
get_logger().debug("removed container", container_id=identifier)
def start_container(self, identifier: str) -> None:
"""Start a container without waiting for it to complete.
:param identifier: Container identifier.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
container.start()
get_logger().debug("started container (detached)", container_id=identifier)
def get_container_status(self, identifier: str) -> str:
"""Get the status of a container.
:param identifier: Container identifier.
:returns: Container status (e.g., "running", "exited", "created").
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
return str(container.status)
def stop_container(self, identifier: str, timeout: int = 10) -> None:
"""Stop a running container gracefully.
:param identifier: Container identifier.
:param timeout: Seconds to wait before killing.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
if container.status == "running":
container.stop(timeout=timeout)
get_logger().debug("stopped container", container_id=identifier)
def read_file_from_container(self, identifier: str, path: str) -> str:
"""Read a file from inside a running container using exec.
:param identifier: Container identifier.
:param path: Path to file inside container.
:returns: File contents as string.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
(status, (stdout, stderr)) = container.exec_run(cmd=["cat", path], demux=True)
if status != 0:
error_msg = stderr.decode("utf-8", errors="replace") if stderr else "File not found"
get_logger().debug("failed to read file from container", path=path, error=error_msg)
return ""
return stdout.decode("utf-8", errors="replace") if stdout else ""
def tail_file_from_container(self, identifier: str, path: str, start_line: int = 1) -> str:
"""Read a file from a container starting at a given line number.
:param identifier: Container identifier.
:param path: Path to file inside container.
:param start_line: 1-based line number to start reading from.
:returns: File contents from *start_line* onwards.
"""
client: PodmanClient = self.get_client()
with client:
container: Container = client.containers.get(key=identifier)
(status, (stdout, stderr)) = container.exec_run(
cmd=["tail", "-n", f"+{start_line}", path],
demux=True,
)
if status != 0:
error_msg = stderr.decode("utf-8", errors="replace") if stderr else "File not found"
get_logger().debug(
"failed to tail file from container", path=path, start_line=start_line, error=error_msg,
)
return ""
return stdout.decode("utf-8", errors="replace") if stdout else ""
def list_containers(self, all_containers: bool = True) -> list[dict]:
"""List containers.
:param all_containers: Include stopped containers.
:returns: List of container info dicts.
"""
client: PodmanClient = self.get_client()
with client:
containers = client.containers.list(all=all_containers)
return [
{
"Id": str(c.id),
"Names": [c.name] if hasattr(c, "name") else [],
"Status": str(c.status),
"Image": str(c.image) if hasattr(c, "image") else "",
}
for c in containers
]
def read_file_from_image(self, image: str, path: str) -> str:
"""Read a file from inside an image without starting a long-running container.
Creates a temporary container, reads the file, and removes the container.
:param image: Image reference (e.g., "secpipe-rust-analyzer:latest").
:param path: Path to file inside image.
:returns: File contents as string.
"""
logger = get_logger()
client: PodmanClient = self.get_client()
with client:
try:
# Create a container that just runs cat on the file
container = client.containers.create(
image=image,
command=["cat", path],
remove=True,
)
# Start it and wait for completion
container.start()
container.wait()
# Get the logs (which contain stdout)
output = container.logs(stdout=True, stderr=False)
if isinstance(output, bytes):
return output.decode("utf-8", errors="replace")
return str(output)
except Exception as exc:
logger.debug("failed to read file from image", image=image, path=path, error=str(exc))
return ""