fuzzforge_ai/backend/benchmarks/category_configs.py

"""
Category-specific benchmark configurations

Defines expected metrics and performance thresholds for each module category.
"""

from dataclasses import dataclass
from typing import List, Dict
from enum import Enum


class ModuleCategory(str, Enum):
    """Module categories for benchmarking"""
    FUZZER = "fuzzer"
    SCANNER = "scanner"
    ANALYZER = "analyzer"
    SECRET_DETECTION = "secret_detection"
    REPORTER = "reporter"


@dataclass
class CategoryBenchmarkConfig:
    """Benchmark configuration for a module category"""
    category: ModuleCategory
    expected_metrics: List[str]
    performance_thresholds: Dict[str, float]
    description: str


# Fuzzer category configuration
FUZZER_CONFIG = CategoryBenchmarkConfig(
    category=ModuleCategory.FUZZER,
    expected_metrics=[
        "execs_per_sec",
        "coverage_rate",
        "time_to_first_crash",
        "corpus_efficiency",
        "execution_time",
        "peak_memory_mb"
    ],
    performance_thresholds={
        "min_execs_per_sec": 1000,  # Minimum executions per second
        "max_execution_time_small": 10.0,  # Max time for small project (seconds)
        "max_execution_time_medium": 60.0,  # Max time for medium project
        "max_memory_mb": 2048,  # Maximum memory usage
        "min_coverage_rate": 1.0,  # Minimum new coverage per second
    },
    description="Fuzzing modules: coverage-guided fuzz testing"
)

# Scanner category configuration
SCANNER_CONFIG = CategoryBenchmarkConfig(
    category=ModuleCategory.SCANNER,
    expected_metrics=[
        "files_per_sec",
        "loc_per_sec",
        "execution_time",
        "peak_memory_mb",
        "findings_count"
    ],
    performance_thresholds={
        "min_files_per_sec": 100,  # Minimum files scanned per second
        "min_loc_per_sec": 10000,  # Minimum lines of code per second
        "max_execution_time_small": 1.0,
        "max_execution_time_medium": 10.0,
        "max_memory_mb": 512,
    },
    description="File scanning modules: fast pattern-based scanning"
)

# Secret detection category configuration
SECRET_DETECTION_CONFIG = CategoryBenchmarkConfig(
    category=ModuleCategory.SECRET_DETECTION,
    expected_metrics=[
        "patterns_per_sec",
        "precision",
        "recall",
        "f1_score",
        "false_positive_rate",
        "execution_time",
        "peak_memory_mb"
    ],
    performance_thresholds={
        "min_patterns_per_sec": 1000,
        "min_precision": 0.90,  # 90% precision target
        "min_recall": 0.95,  # 95% recall target
        "max_false_positives": 5,  # Max false positives per 100 secrets
        "max_execution_time_small": 2.0,
        "max_execution_time_medium": 20.0,
        "max_memory_mb": 1024,
    },
    description="Secret detection modules: high precision pattern matching"
)

# Analyzer category configuration
ANALYZER_CONFIG = CategoryBenchmarkConfig(
    category=ModuleCategory.ANALYZER,
    expected_metrics=[
        "analysis_depth",
        "files_analyzed_per_sec",
        "execution_time",
        "peak_memory_mb",
        "findings_count",
        "accuracy"
    ],
    performance_thresholds={
        "min_files_per_sec": 10,  # Slower than scanners due to deep analysis
        "max_execution_time_small": 5.0,
        "max_execution_time_medium": 60.0,
        "max_memory_mb": 2048,
        "min_accuracy": 0.85,  # 85% accuracy target
    },
    description="Code analysis modules: deep semantic analysis"
)

# Reporter category configuration
REPORTER_CONFIG = CategoryBenchmarkConfig(
    category=ModuleCategory.REPORTER,
    expected_metrics=[
        "report_generation_time",
        "findings_per_sec",
        "peak_memory_mb"
    ],
    performance_thresholds={
        "max_report_time_100_findings": 1.0,  # Max 1 second for 100 findings
        "max_report_time_1000_findings": 10.0,  # Max 10 seconds for 1000 findings
        "max_memory_mb": 256,
    },
    description="Reporting modules: fast report generation"
)


# Category configurations map
CATEGORY_CONFIGS = {
    ModuleCategory.FUZZER: FUZZER_CONFIG,
    ModuleCategory.SCANNER: SCANNER_CONFIG,
    ModuleCategory.SECRET_DETECTION: SECRET_DETECTION_CONFIG,
    ModuleCategory.ANALYZER: ANALYZER_CONFIG,
    ModuleCategory.REPORTER: REPORTER_CONFIG,
}


def get_category_config(category: ModuleCategory) -> CategoryBenchmarkConfig:
    """Get benchmark configuration for a category"""
    return CATEGORY_CONFIGS[category]


def get_threshold(category: ModuleCategory, metric: str) -> float:
    """Get performance threshold for a specific metric"""
    config = get_category_config(category)
    return config.performance_thresholds.get(metric, 0.0)