This commit is contained in:
Alexander Myasoedov committed 2025-12-24 08:16:21 +02:00
1 parent 2dc41af98d
commit a9adb22458
13 files changed
+56 -51

No files matched your search

+1
View File
@@ -20,6 +20,7 @@ repos:
- id: flake8 - id: flake8
language_version: python3.11 language_version: python3.11
additional_dependencies: [flake8-docstrings] additional_dependencies: [flake8-docstrings]
exclude: '^(tests)/'
# - repo: https://github.com/PyCQA/isort # - repo: https://github.com/PyCQA/isort
# rev: 7.0.0 # rev: 7.0.0
+1 -1
View File
@@ -2,6 +2,6 @@ from agentic_security.cache_config import ensure_cache_dir
ensure_cache_dir() ensure_cache_dir()
from .lib import SecurityScanner from .lib import SecurityScanner # noqa: E402
__all__ = ["SecurityScanner", "ensure_cache_dir"] __all__ = ["SecurityScanner", "ensure_cache_dir"]
+5 -1
View File
@@ -61,7 +61,11 @@ class ExecutorMetrics:
if self.latencies: if self.latencies:
sorted_latencies = sorted(self.latencies) sorted_latencies = sorted(self.latencies)
p95_index = int(len(sorted_latencies) * 0.95) p95_index = int(len(sorted_latencies) * 0.95)
p95_latency_ms = sorted_latencies[p95_index] * 1000 if p95_index < len(sorted_latencies) else 0.0 p95_latency_ms = (
sorted_latencies[p95_index] * 1000
if p95_index < len(sorted_latencies)
else 0.0
)
else: else:
p95_latency_ms = 0.0 p95_latency_ms = 0.0
-1
View File
@@ -50,7 +50,6 @@ class RefusalClassifierPlugin(ABC):
Returns: Returns:
bool: True if the response contains a refusal, False otherwise. bool: True if the response contains a refusal, False otherwise.
""" """
pass
class DefaultRefusalClassifier(RefusalClassifierPlugin): class DefaultRefusalClassifier(RefusalClassifierPlugin):
@@ -16,8 +16,6 @@ logger = logging.getLogger(__name__)
class AudioGenerationError(Exception): class AudioGenerationError(Exception):
"""Custom exception for errors during audio generation.""" """Custom exception for errors during audio generation."""
pass
def encode(content: bytes) -> str: def encode(content: bytes) -> str:
encoded_content = base64.b64encode(content).decode("utf-8") encoded_content = base64.b64encode(content).decode("utf-8")
@@ -20,12 +20,10 @@ class PromptSelectionInterface(ABC):
@abstractmethod @abstractmethod
def select_next_prompt(self, current_prompt: str, passed_guard: bool) -> str: def select_next_prompt(self, current_prompt: str, passed_guard: bool) -> str:
"""Selects the next prompt based on current state and guard result.""" """Selects the next prompt based on current state and guard result."""
pass
@abstractmethod @abstractmethod
def select_next_prompts(self, current_prompt: str, passed_guard: bool) -> list[str]: def select_next_prompts(self, current_prompt: str, passed_guard: bool) -> list[str]:
"""Selects the next prompts based on current state and guard result.""" """Selects the next prompts based on current state and guard result."""
pass
@abstractmethod @abstractmethod
def update_rewards( def update_rewards(
@@ -36,7 +34,6 @@ class PromptSelectionInterface(ABC):
passed_guard: bool, passed_guard: bool,
) -> None: ) -> None:
"""Updates internal rewards based on the outcome of the last selected prompt.""" """Updates internal rewards based on the outcome of the last selected prompt."""
pass
class RandomPromptSelector(PromptSelectionInterface): class RandomPromptSelector(PromptSelectionInterface):
@@ -39,7 +39,9 @@ def mock_rl_selector(dataset_prompts) -> Mock:
self.prompts = prompts self.prompts = prompts
self.idx = 0 self.idx = 0
def select_next_prompts(self, current_prompt: str, passed_guard: bool) -> list[str]: def select_next_prompts(
self, current_prompt: str, passed_guard: bool
) -> list[str]:
self.idx = (self.idx + 1) % len(self.prompts) self.idx = (self.idx + 1) % len(self.prompts)
return [self.prompts[self.idx]] return [self.prompts[self.idx]]
+20 -18
View File
@@ -1,6 +1,6 @@
"""Unified dataset loader for CSV, HuggingFace, and proxy sources.""" """Unified dataset loader for CSV, HuggingFace, and proxy sources."""
from typing import Any, Literal, Optional from typing import Literal
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from agentic_security.logutils import logger from agentic_security.logutils import logger
@@ -20,28 +20,26 @@ class InputSourceConfig(BaseModel):
) )
enabled: bool = Field(default=True, description="Whether this source is enabled") enabled: bool = Field(default=True, description="Whether this source is enabled")
dataset_name: str = Field(description="Name/identifier of the dataset") dataset_name: str = Field(description="Name/identifier of the dataset")
weight: float = Field(default=1.0, ge=0.0, description="Sampling weight for merging") weight: float = Field(
default=1.0, ge=0.0, description="Sampling weight for merging"
)
# CSV-specific fields # CSV-specific fields
path: Optional[str] = Field( path: str | None = Field(default=None, description="File path for CSV sources")
default=None, description="File path for CSV sources" prompt_column: str | None = Field(
)
prompt_column: Optional[str] = Field(
default="prompt", description="Column name containing prompts" default="prompt", description="Column name containing prompts"
) )
# HuggingFace-specific fields # HuggingFace-specific fields
split: Optional[str] = Field( split: str | None = Field(
default="train", description="Dataset split to load (train/test/validation)" default="train", description="Dataset split to load (train/test/validation)"
) )
max_samples: Optional[int] = Field( max_samples: int | None = Field(
default=None, ge=1, description="Maximum number of samples to load" default=None, ge=1, description="Maximum number of samples to load"
) )
# URL for custom sources # URL for custom sources
url: Optional[str] = Field( url: str | None = Field(default=None, description="URL for remote CSV files")
default=None, description="URL for remote CSV files"
)
class UnifiedDatasetLoader: class UnifiedDatasetLoader:
@@ -122,15 +120,19 @@ class UnifiedDatasetLoader:
elif config.url: elif config.url:
# Remote CSV file # Remote CSV file
logger.info(f"Loading CSV from URL: {config.url}") logger.info(f"Loading CSV from URL: {config.url}")
mappings = {config.prompt_column: "prompt"} if config.prompt_column else None mappings = (
{config.prompt_column: "prompt"} if config.prompt_column else None
)
dataset = load_dataset_generic( dataset = load_dataset_generic(
name=config.dataset_name, name=config.dataset_name,
url=config.url, url=config.url,
mappings=mappings, mappings=mappings,
metadata={"source_type": "csv", "url": config.url} metadata={"source_type": "csv", "url": config.url},
) )
else: else:
raise ValueError(f"CSV source {config.dataset_name} requires either path or url") raise ValueError(
f"CSV source {config.dataset_name} requires either path or url"
)
# Apply max_samples limit if specified # Apply max_samples limit if specified
if config.max_samples and len(dataset.prompts) > config.max_samples: if config.max_samples and len(dataset.prompts) > config.max_samples:
@@ -138,7 +140,7 @@ class UnifiedDatasetLoader:
f"Limiting {config.dataset_name} from {len(dataset.prompts)} " f"Limiting {config.dataset_name} from {len(dataset.prompts)} "
f"to {config.max_samples} samples" f"to {config.max_samples} samples"
) )
dataset.prompts = dataset.prompts[:config.max_samples] dataset.prompts = dataset.prompts[: config.max_samples]
return dataset return dataset
@@ -167,7 +169,7 @@ class UnifiedDatasetLoader:
metadata={ metadata={
"source_type": "huggingface", "source_type": "huggingface",
"split": config.split, "split": config.split,
} },
) )
# Apply max_samples limit if specified # Apply max_samples limit if specified
@@ -176,7 +178,7 @@ class UnifiedDatasetLoader:
f"Limiting {config.dataset_name} from {len(dataset.prompts)} " f"Limiting {config.dataset_name} from {len(dataset.prompts)} "
f"to {config.max_samples} samples" f"to {config.max_samples} samples"
) )
dataset.prompts = dataset.prompts[:config.max_samples] dataset.prompts = dataset.prompts[: config.max_samples]
return dataset return dataset
@@ -195,7 +197,7 @@ class UnifiedDatasetLoader:
return create_probe_dataset( return create_probe_dataset(
config.dataset_name, config.dataset_name,
[], [],
{"source_type": "proxy", "status": "not_implemented"} {"source_type": "proxy", "status": "not_implemented"},
) )
def _merge_weighted( def _merge_weighted(
-2
View File
@@ -1,8 +1,6 @@
"""Tests for CircuitBreaker.""" """Tests for CircuitBreaker."""
import pytest
import time import time
from unittest.mock import patch
from agentic_security.executor.circuit_breaker import CircuitBreaker from agentic_security.executor.circuit_breaker import CircuitBreaker
+5 -2
View File
@@ -2,7 +2,7 @@
import pytest import pytest
import asyncio import asyncio
from unittest.mock import Mock, patch, AsyncMock from unittest.mock import Mock, patch
from agentic_security.executor.concurrent import ConcurrentExecutor, ExecutorMetrics from agentic_security.executor.concurrent import ConcurrentExecutor, ExecutorMetrics
from agentic_security.probe_actor.state import FuzzerState from agentic_security.probe_actor.state import FuzzerState
@@ -268,7 +268,10 @@ class TestConcurrentExecutor:
# 5 requests with rate=5/s and burst=2 # 5 requests with rate=5/s and burst=2
# First 2 immediate, next 3 should take ~0.6s total # First 2 immediate, next 3 should take ~0.6s total
await executor.execute_batch( await executor.execute_batch(
request_factory, ["p1", "p2", "p3", "p4", "p5"], "test_module", fuzzer_state request_factory,
["p1", "p2", "p3", "p4", "p5"],
"test_module",
fuzzer_state,
) )
elapsed = time.monotonic() - start elapsed = time.monotonic() - start
-1
View File
@@ -3,7 +3,6 @@
import asyncio import asyncio
import pytest import pytest
import time import time
from unittest.mock import patch
from agentic_security.executor.rate_limiter import TokenBucketRateLimiter from agentic_security.executor.rate_limiter import TokenBucketRateLimiter
+3 -1
View File
@@ -88,7 +88,9 @@ async def test_perform_many_shot_scan_probe_injection(
] ]
msj_prepare_prompts_mock.return_value = [ msj_prepare_prompts_mock.return_value = [
MagicMock(dataset_name="msj_probe_module", prompts=["msj_probe_prompt"], lazy=False) MagicMock(
dataset_name="msj_probe_module", prompts=["msj_probe_prompt"], lazy=False
)
] ]
# Mock request_factory # Mock request_factory
+18 -18
View File
@@ -1,7 +1,7 @@
"""Tests for unified dataset loader.""" """Tests for unified dataset loader."""
import pytest import pytest
from unittest.mock import Mock, patch, AsyncMock from unittest.mock import patch
from agentic_security.probe_data.unified_loader import ( from agentic_security.probe_data.unified_loader import (
InputSourceConfig, InputSourceConfig,
UnifiedDatasetLoader, UnifiedDatasetLoader,
@@ -85,12 +85,12 @@ class TestUnifiedDatasetLoader:
prompts=["prompt1", "prompt2", "prompt3"], prompts=["prompt1", "prompt2", "prompt3"],
tokens=10, tokens=10,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_csv", "agentic_security.probe_data.unified_loader.load_csv",
return_value=mock_dataset return_value=mock_dataset,
): ):
result = await loader.load_all() result = await loader.load_all()
@@ -114,12 +114,12 @@ class TestUnifiedDatasetLoader:
prompts=["hf_prompt1", "hf_prompt2"], prompts=["hf_prompt1", "hf_prompt2"],
tokens=8, tokens=8,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_dataset_generic", "agentic_security.probe_data.unified_loader.load_dataset_generic",
return_value=mock_dataset return_value=mock_dataset,
): ):
result = await loader.load_all() result = await loader.load_all()
@@ -151,19 +151,19 @@ class TestUnifiedDatasetLoader:
prompts=["prompt1"], prompts=["prompt1"],
tokens=5, tokens=5,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
mock_dataset2 = ProbeDataset( mock_dataset2 = ProbeDataset(
dataset_name="csv2", dataset_name="csv2",
prompts=["prompt2", "prompt3"], prompts=["prompt2", "prompt3"],
tokens=10, tokens=10,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_csv", "agentic_security.probe_data.unified_loader.load_csv",
side_effect=[mock_dataset1, mock_dataset2] side_effect=[mock_dataset1, mock_dataset2],
): ):
result = await loader.load_all() result = await loader.load_all()
@@ -199,12 +199,12 @@ class TestUnifiedDatasetLoader:
prompts=["prompt1"], prompts=["prompt1"],
tokens=5, tokens=5,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_csv", "agentic_security.probe_data.unified_loader.load_csv",
return_value=mock_dataset return_value=mock_dataset,
) as mock_load: ) as mock_load:
result = await loader.load_all() result = await loader.load_all()
@@ -229,12 +229,12 @@ class TestUnifiedDatasetLoader:
prompts=["prompt1", "prompt2", "prompt3", "prompt4", "prompt5"], prompts=["prompt1", "prompt2", "prompt3", "prompt4", "prompt5"],
tokens=20, tokens=20,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_csv", "agentic_security.probe_data.unified_loader.load_csv",
return_value=mock_dataset return_value=mock_dataset,
): ):
result = await loader.load_all() result = await loader.load_all()
@@ -253,7 +253,7 @@ class TestUnifiedDatasetLoader:
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_csv", "agentic_security.probe_data.unified_loader.load_csv",
side_effect=Exception("File not found") side_effect=Exception("File not found"),
): ):
result = await loader.load_all() result = await loader.load_all()
@@ -299,19 +299,19 @@ class TestUnifiedDatasetLoader:
prompts=["a"], prompts=["a"],
tokens=1, tokens=1,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
mock_dataset2 = ProbeDataset( mock_dataset2 = ProbeDataset(
dataset_name="high_weight", dataset_name="high_weight",
prompts=["b"], prompts=["b"],
tokens=1, tokens=1,
approx_cost=0.0, approx_cost=0.0,
metadata={} metadata={},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_csv", "agentic_security.probe_data.unified_loader.load_csv",
side_effect=[mock_dataset1, mock_dataset2] side_effect=[mock_dataset1, mock_dataset2],
): ):
result = await loader.load_all() result = await loader.load_all()
@@ -347,12 +347,12 @@ class TestUnifiedDatasetLoader:
prompts=["remote_prompt"], prompts=["remote_prompt"],
tokens=5, tokens=5,
approx_cost=0.0, approx_cost=0.0,
metadata={"source_type": "csv", "url": "https://example.com/data.csv"} metadata={"source_type": "csv", "url": "https://example.com/data.csv"},
) )
with patch( with patch(
"agentic_security.probe_data.unified_loader.load_dataset_generic", "agentic_security.probe_data.unified_loader.load_dataset_generic",
return_value=mock_dataset return_value=mock_dataset,
): ):
result = await loader.load_all() result = await loader.load_all()