mirror of
https://github.com/FuzzingLabs/fuzzforge_ai.git
synced 2026-10-13 23:30:52 +02:00
feat: Complete migration from Prefect to Temporal
BREAKING CHANGE: Replaces Prefect workflow orchestration with Temporal ## Major Changes - Replace Prefect with Temporal for workflow orchestration - Implement vertical worker architecture (rust, android) - Replace Docker registry with MinIO for unified storage - Refactor activities to be co-located with workflows - Update all API endpoints for Temporal compatibility ## Infrastructure - New: docker-compose.temporal.yaml (Temporal + MinIO + workers) - New: workers/ directory with rust and android vertical workers - New: backend/src/temporal/ (manager, discovery) - New: backend/src/storage/ (S3-cached storage with MinIO) - New: backend/toolbox/common/ (shared storage activities) - Deleted: docker-compose.yaml (old Prefect setup) - Deleted: backend/src/core/prefect_manager.py - Deleted: backend/src/services/prefect_stats_monitor.py - Deleted: Docker registry and insecure-registries requirement ## Workflows - Migrated: security_assessment workflow to Temporal - New: rust_test workflow (example/test workflow) - Deleted: secret_detection_scan (Prefect-based, to be reimplemented) - Activities now co-located with workflows for independent testing ## API Changes - Updated: backend/src/api/workflows.py (Temporal submission) - Updated: backend/src/api/runs.py (Temporal status/results) - Updated: backend/src/main.py (727 lines, TemporalManager integration) - Updated: All 16 MCP tools to use TemporalManager ## Testing - ✅ All services healthy (Temporal, PostgreSQL, MinIO, workers, backend) - ✅ All API endpoints functional - ✅ End-to-end workflow test passed (72 findings from vulnerable_app) - ✅ MinIO storage integration working (target upload/download, results) - ✅ Worker activity discovery working (6 activities registered) - ✅ Tarball extraction working - ✅ SARIF report generation working ## Documentation - ARCHITECTURE.md: Complete Temporal architecture documentation - QUICKSTART_TEMPORAL.md: Getting started guide - MIGRATION_DECISION.md: Why we chose Temporal over Prefect - IMPLEMENTATION_STATUS.md: Migration progress tracking - workers/README.md: Worker development guide ## Dependencies - Added: temporalio>=1.6.0 - Added: boto3>=1.34.0 (MinIO S3 client) - Removed: prefect>=3.4.18
This commit is contained in:
1 parent
987c49569c
commit
0680f14df6
48 files changed
+7504
-3101
No files matched your search
@@ -0,0 +1,21 @@
|
||||
name: rust_test
|
||||
version: 1.0.0
|
||||
description: "Simple test workflow for Rust vertical worker"
|
||||
vertical: rust
|
||||
author: "FuzzForge Team"
|
||||
tags:
|
||||
- test
|
||||
- rust
|
||||
- example
|
||||
dependencies:
|
||||
python: []
|
||||
parameters:
|
||||
- name: target_id
|
||||
type: string
|
||||
required: true
|
||||
description: "UUID of the uploaded target in MinIO"
|
||||
- name: test_message
|
||||
type: string
|
||||
required: false
|
||||
default: "Hello from Rust workflow!"
|
||||
description: "Test message to include in results"
|
||||
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
Rust Test Workflow
|
||||
|
||||
Simple test workflow to verify:
|
||||
1. Temporal worker discovery works
|
||||
2. MinIO storage integration works
|
||||
3. Activities execute correctly
|
||||
4. Results are properly returned
|
||||
|
||||
This workflow:
|
||||
- Downloads a target from MinIO
|
||||
- Performs a simple analysis (file inspection)
|
||||
- Returns results
|
||||
- Cleans up cache
|
||||
"""
|
||||
|
||||
from datetime import timedelta
|
||||
from typing import Optional
|
||||
|
||||
from temporalio import workflow
|
||||
from temporalio.common import RetryPolicy
|
||||
|
||||
# Import activity interfaces (will be executed by worker)
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@workflow.defn
|
||||
class RustTestWorkflow:
|
||||
"""
|
||||
Simple test workflow for Rust vertical.
|
||||
|
||||
This demonstrates the basic workflow pattern:
|
||||
1. Download target from MinIO
|
||||
2. Execute activities
|
||||
3. Return results
|
||||
4. Cleanup
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(
|
||||
self,
|
||||
target_id: str,
|
||||
test_message: Optional[str] = "Hello from Rust workflow!"
|
||||
) -> dict:
|
||||
"""
|
||||
Main workflow execution.
|
||||
|
||||
Args:
|
||||
target_id: UUID of the uploaded target in MinIO
|
||||
test_message: Optional test message to include in results
|
||||
|
||||
Returns:
|
||||
Dictionary containing workflow results
|
||||
"""
|
||||
workflow_id = workflow.info().workflow_id
|
||||
|
||||
workflow.logger.info(
|
||||
f"Starting RustTestWorkflow "
|
||||
f"(workflow_id={workflow_id}, target_id={target_id})"
|
||||
)
|
||||
|
||||
results = {
|
||||
"workflow_id": workflow_id,
|
||||
"target_id": target_id,
|
||||
"message": test_message,
|
||||
"steps": []
|
||||
}
|
||||
|
||||
try:
|
||||
# Step 1: Download target from MinIO
|
||||
workflow.logger.info("Step 1: Downloading target from MinIO")
|
||||
target_path = await workflow.execute_activity(
|
||||
"get_target",
|
||||
target_id,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
retry_policy=RetryPolicy(
|
||||
initial_interval=timedelta(seconds=1),
|
||||
maximum_interval=timedelta(seconds=30),
|
||||
maximum_attempts=3
|
||||
)
|
||||
)
|
||||
results["steps"].append({
|
||||
"step": "download_target",
|
||||
"status": "success",
|
||||
"target_path": target_path
|
||||
})
|
||||
workflow.logger.info(f"✓ Target downloaded to: {target_path}")
|
||||
|
||||
# Step 2: Perform simple analysis (inline for testing)
|
||||
workflow.logger.info("Step 2: Performing simple analysis")
|
||||
# In a real workflow, this would be an activity that uses
|
||||
# AFL++, cargo-fuzz, or other Rust tools
|
||||
|
||||
analysis_result = {
|
||||
"file_path": target_path,
|
||||
"analysis_type": "test",
|
||||
"findings": [
|
||||
{
|
||||
"type": "info",
|
||||
"message": "Test workflow executed successfully",
|
||||
"test_message": test_message
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
results["steps"].append({
|
||||
"step": "analysis",
|
||||
"status": "success",
|
||||
"analysis": analysis_result
|
||||
})
|
||||
workflow.logger.info("✓ Analysis completed")
|
||||
|
||||
# Step 3: Upload results to MinIO (optional)
|
||||
workflow.logger.info("Step 3: Uploading results")
|
||||
try:
|
||||
results_url = await workflow.execute_activity(
|
||||
"upload_results",
|
||||
args=[workflow_id, results, "json"],
|
||||
start_to_close_timeout=timedelta(minutes=2)
|
||||
)
|
||||
results["results_url"] = results_url
|
||||
workflow.logger.info(f"✓ Results uploaded to: {results_url}")
|
||||
except Exception as e:
|
||||
workflow.logger.warning(f"Failed to upload results: {e}")
|
||||
# Don't fail workflow if upload fails
|
||||
results["results_url"] = None
|
||||
|
||||
# Step 4: Cleanup cache
|
||||
workflow.logger.info("Step 4: Cleaning up cache")
|
||||
try:
|
||||
await workflow.execute_activity(
|
||||
"cleanup_cache",
|
||||
target_path,
|
||||
start_to_close_timeout=timedelta(minutes=1)
|
||||
)
|
||||
workflow.logger.info("✓ Cache cleaned up")
|
||||
except Exception as e:
|
||||
workflow.logger.warning(f"Cache cleanup failed: {e}")
|
||||
# Don't fail workflow if cleanup fails
|
||||
|
||||
# Mark workflow as successful
|
||||
results["status"] = "success"
|
||||
workflow.logger.info(f"✓ Workflow completed successfully: {workflow_id}")
|
||||
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
workflow.logger.error(f"Workflow failed: {e}")
|
||||
results["status"] = "error"
|
||||
results["error"] = str(e)
|
||||
results["steps"].append({
|
||||
"step": "error",
|
||||
"status": "failed",
|
||||
"error": str(e)
|
||||
})
|
||||
raise
|
||||
Reference in new issue
Block a user