mirror of
https://github.com/FuzzingLabs/fuzzforge_ai.git
synced 2026-10-12 04:17:06 +02:00
feat: Complete migration from Prefect to Temporal
BREAKING CHANGE: Replaces Prefect workflow orchestration with Temporal ## Major Changes - Replace Prefect with Temporal for workflow orchestration - Implement vertical worker architecture (rust, android) - Replace Docker registry with MinIO for unified storage - Refactor activities to be co-located with workflows - Update all API endpoints for Temporal compatibility ## Infrastructure - New: docker-compose.temporal.yaml (Temporal + MinIO + workers) - New: workers/ directory with rust and android vertical workers - New: backend/src/temporal/ (manager, discovery) - New: backend/src/storage/ (S3-cached storage with MinIO) - New: backend/toolbox/common/ (shared storage activities) - Deleted: docker-compose.yaml (old Prefect setup) - Deleted: backend/src/core/prefect_manager.py - Deleted: backend/src/services/prefect_stats_monitor.py - Deleted: Docker registry and insecure-registries requirement ## Workflows - Migrated: security_assessment workflow to Temporal - New: rust_test workflow (example/test workflow) - Deleted: secret_detection_scan (Prefect-based, to be reimplemented) - Activities now co-located with workflows for independent testing ## API Changes - Updated: backend/src/api/workflows.py (Temporal submission) - Updated: backend/src/api/runs.py (Temporal status/results) - Updated: backend/src/main.py (727 lines, TemporalManager integration) - Updated: All 16 MCP tools to use TemporalManager ## Testing - ✅ All services healthy (Temporal, PostgreSQL, MinIO, workers, backend) - ✅ All API endpoints functional - ✅ End-to-end workflow test passed (72 findings from vulnerable_app) - ✅ MinIO storage integration working (target upload/download, results) - ✅ Worker activity discovery working (6 activities registered) - ✅ Tarball extraction working - ✅ SARIF report generation working ## Documentation - ARCHITECTURE.md: Complete Temporal architecture documentation - QUICKSTART_TEMPORAL.md: Getting started guide - MIGRATION_DECISION.md: Why we chose Temporal over Prefect - IMPLEMENTATION_STATUS.md: Migration progress tracking - workers/README.md: Worker development guide ## Dependencies - Added: temporalio>=1.6.0 - Added: boto3>=1.34.0 (MinIO S3 client) - Removed: prefect>=3.4.18
This commit is contained in:
1 parent
987c49569c
commit
0680f14df6
48 files changed
+7504
-3101
No files matched your search
@@ -1,82 +0,0 @@
|
||||
# Copyright (c) 2025 FuzzingLabs
|
||||
#
|
||||
# Licensed under the Business Source License 1.1 (BSL). See the LICENSE file
|
||||
# at the root of this repository for details.
|
||||
#
|
||||
# After the Change Date (four years from publication), this version of the
|
||||
# Licensed Work will be made available under the Apache License, Version 2.0.
|
||||
# See the LICENSE-APACHE file or http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Additional attribution and requirements are provided in the NOTICE file.
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
|
||||
from src.services.prefect_stats_monitor import PrefectStatsMonitor
|
||||
from src.api import fuzzing
|
||||
|
||||
|
||||
class FakeLog:
|
||||
def __init__(self, message: str):
|
||||
self.message = message
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, logs):
|
||||
self._logs = logs
|
||||
|
||||
async def read_logs(self, log_filter=None, limit=100, sort="TIMESTAMP_ASC"):
|
||||
return self._logs
|
||||
|
||||
|
||||
class FakeTaskRun:
|
||||
def __init__(self):
|
||||
self.id = "task-1"
|
||||
self.start_time = datetime.now(timezone.utc) - timedelta(seconds=5)
|
||||
|
||||
|
||||
def test_parse_stats_from_log_fuzzing():
|
||||
mon = PrefectStatsMonitor()
|
||||
msg = (
|
||||
"INFO LIVE_STATS extra={'stats_type': 'fuzzing_live_update', "
|
||||
"'executions': 42, 'executions_per_sec': 3.14, 'crashes': 1, 'unique_crashes': 1, 'corpus_size': 9}"
|
||||
)
|
||||
stats = mon._parse_stats_from_log(msg)
|
||||
assert stats is not None
|
||||
assert stats["stats_type"] == "fuzzing_live_update"
|
||||
assert stats["executions"] == 42
|
||||
|
||||
|
||||
def test_extract_stats_updates_and_broadcasts():
|
||||
mon = PrefectStatsMonitor()
|
||||
run_id = "run-123"
|
||||
workflow = "wf"
|
||||
fuzzing.initialize_fuzzing_tracking(run_id, workflow)
|
||||
|
||||
# Prepare a fake websocket to capture messages
|
||||
sent = []
|
||||
|
||||
class FakeWS:
|
||||
async def send_text(self, text: str):
|
||||
sent.append(text)
|
||||
|
||||
fuzzing.active_connections[run_id] = [FakeWS()]
|
||||
|
||||
# Craft a log line the parser understands
|
||||
msg = (
|
||||
"INFO LIVE_STATS extra={'stats_type': 'fuzzing_live_update', "
|
||||
"'executions': 10, 'executions_per_sec': 1.5, 'crashes': 0, 'unique_crashes': 0, 'corpus_size': 2}"
|
||||
)
|
||||
fake_client = FakeClient([FakeLog(msg)])
|
||||
task_run = FakeTaskRun()
|
||||
|
||||
asyncio.run(mon._extract_stats_from_task(fake_client, run_id, task_run, workflow))
|
||||
|
||||
# Verify stats updated
|
||||
stats = fuzzing.fuzzing_stats[run_id]
|
||||
assert stats.executions == 10
|
||||
assert stats.executions_per_sec == 1.5
|
||||
|
||||
# Verify a message was sent to WebSocket
|
||||
assert sent, "Expected a stats_update message to be sent"
|
||||
Reference in new issue
Block a user