feat(bump version):

Add missing documentation in report_chart.py

.gitignore

@@ -17,3 +17,4 @@ inv/
 scripts/
 docx/
 agentic_security.toml
+/venv

Readme.md

@@ -1,37 +1,54 @@
 <p align="center">
-
-<h1 align="center">Agentic Security</h1>
-
-<p align="center">
-    The open-source Agentic LLM Vulnerability Scanner
-    <br />
-    <br />
+  <h1 align="center">Agentic Security</h1>
+  <p align="center">
+    An open-source vulnerability scanner for Agent Workflows and Large Language Models (LLMs)<br />
+    Protecting AI systems from jailbreaks, fuzzing, and multimodal attacks.<br />
+    <a href="https://agentic-security.vercel.app">Explore the docs »</a> ·
+    <a href="https://github.com/msoedov/agentic_security/issues">Report a Bug »</a>
+  </p>
 </p>
 
 <p align="center">
   <a href="https://github.com/msoedov/agentic_security/commits/main">
-    <img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/msoedov/agentic_security?style=for-the-badge&logo=git&labelColor=000000&logoColor=FFFFFF&label=Last Commit&color=6A35FF" />
+    <img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/msoedov/agentic_security?style=for-the-badge&logo=git&labelColor=000000&color=6A35FF" />
   </a>
   <a href="https://github.com/msoedov/agentic_security">
-    <img alt="GitHub Repo Size" src="https://img.shields.io/github/repo-size/msoedov/agentic_security?style=for-the-badge&logo=database&labelColor=000000&logoColor=FFFFFF&label=Repo Size&color=yellow" />
-  </a>
+    <img alt="GitHub Repo Size" src="https://img.shields.io/github/repo-size/msoedov/agentic_security?style=for-the-badge&logo=database&labelColor=000000&color=yellow" />
   </a>
   <a href="https://github.com/msoedov/agentic_security/blob/master/LICENSE">
-    <img alt="GitHub License" src="https://img.shields.io/github/license/msoedov/agentic_security?style=for-the-badge&logo=codeigniter&labelColor=000000&logoColor=FFFFFF&label=License&color=FFCC19" />
+    <img alt="GitHub License" src="https://img.shields.io/github/license/msoedov/agentic_security?style=for-the-badge&logo=codeigniter&labelColor=000000&color=FFCC19" />
+  </a>
+  <a href="https://pypi.org/project/agentic-security/">
+    <img alt="PyPI Version" src="https://img.shields.io/pypi/v/agentic-security?style=for-the-badge&logo=pypi&labelColor=000000&color=00CCFF" />
+  </a>
+  <a href="https://discord.gg/stw3DfZQ">
+    <img alt="Join Discord" src="https://img.shields.io/badge/Discord-Join%20Us-black?style=for-the-badge&logo=discord&labelColor=000000&color=DD55FF" />
   </a>
-  <a href="https://discord.gg/stw3DfZQ"><img alt="Join the community" src="https://img.shields.io/badge/Join%20the%20community-black.svg?style=for-the-badge&logo=lightning&labelColor=000000&logoColor=FFFFFF&label=&color=DD55FF&logoWidth=20" /></a>
-
 </p>
 
+
 ## Features
 
-- Multi modal attacks and vulnerability scanners🛠️
-- Multi-Step/multi-round Jailbreaks 🌀
-- Comprehensive fuzzing for any LLMs 🧪
-- LLM API integration and stress testing 🛠️
-- RL based attacks 📡
 
-Note: Please be aware that Agentic Security is designed as a safety scanner tool and not a foolproof solution. It cannot guarantee complete protection against all possible threats.
+Agentic Security equips you with powerful tools to safeguard LLMs against emerging threats. Here's what you can do:
+
+- **Multimodal Attacks** 🖼️🎙️
+  Probe vulnerabilities across text, images, and audio inputs to ensure your LLM is robust against diverse threats.
+
+- **Multi-Step Jailbreaks** 🌀
+  Simulate sophisticated, iterative attack sequences to uncover weaknesses in LLM safety mechanisms.
+
+- **Comprehensive Fuzzing** 🧪
+  Stress-test any LLM with randomized inputs to identify edge cases and unexpected behaviors.
+
+- **API Integration & Stress Testing** 🌐
+  Seamlessly connect to LLM APIs and push their limits with high-volume, real-world attack scenarios.
+
+- **RL-Based Attacks** 📡
+  Leverage reinforcement learning to craft adaptive, intelligent probes that evolve with your model’s defenses.
+
+> **Why It Matters**: These features help developers, researchers, and security teams proactively identify and mitigate risks in AI systems, ensuring safer and more reliable deployments.
+
 
 ## 📦 Installation
 
@@ -66,7 +83,7 @@ agentic_security --port=PORT --host=HOST
 
 ## UI 🧙
 
-<img width="100%" alt="booking-screen" src="https://res.cloudinary.com/dq0w2rtm9/image/upload/v1736433557/z0bsyzhsqlgcr3w4ovwp.gif">
+<img width="100%" alt="booking-screen" src="https://raw.githubusercontent.com/msoedov/agentic_security/refs/heads/main/docs/images/demo.gif">
 
 ## LLM kwargs
 
@@ -111,7 +128,7 @@ Init config
 ```shell
 agentic_security init
 
-2025-01-08 20:12:02.449 | INFO     | agentic_security.lib:generate_default_cfg:324 - Default configuration generated successfully to agesec.toml.
+2025-01-08 20:12:02.449 | INFO     | agentic_security.lib:generate_default_settings:324 - Default configuration generated successfully to agesec.toml.
 
 ```
 
@@ -391,10 +408,15 @@ For more detailed information on how to use Agentic Security, including advanced
 
 ## Roadmap and Future Goals
 
-- \[ \] Expand dataset variety
-- \[ \] Introduce two new attack vectors
-- \[ \] Develop initial attacker LLM
-- \[ \] Complete integration of OWASP Top 10 classification
+
+
+We’re just getting started! Here’s what’s on the horizon:
+
+- **RL-Powered Attacks**: An attacker LLM trained with reinforcement learning to dynamically evolve jailbreaks and outsmart defenses.
+- **Massive Dataset Expansion**: Scaling to 100,000+ prompts across text, image, and audio modalities—curated for real-world threats.
+- **Daily Attack Updates**: Fresh attack vectors delivered daily, keeping your scans ahead of the curve.
+- **Community Modules**: A plug-and-play ecosystem where you can share and deploy custom probes, datasets, and integrations.
+
 
 | Tool                    | Source                                                                        | Integrated |
 |-------------------------|-------------------------------------------------------------------------------|------------|
@@ -422,4 +444,9 @@ Before contributing, please read the contributing guidelines.
 
 Agentic Security is released under the Apache License v2.
 
+
+## 🚫 No Cryptocurrency Affiliation
+
+Agentic Security is focused solely on AI security and has no affiliation with cryptocurrency projects, blockchain technologies, or related initiatives. Our mission is to advance the safety and reliability of AI systems—no tokens, no coins, just code.
+
 ## Contact us

agentic_security/__init__.py

@@ -1,3 +1,3 @@
-from .lib import AgenticSecurity
+from .lib import SecurityScanner
 
-__all__ = ["AgenticSecurity"]
+__all__ = ["SecurityScanner"]

agentic_security/__main__.py

@@ -5,7 +5,8 @@ import fire
 import uvicorn
 
 from agentic_security.app import app
-from agentic_security.lib import AgenticSecurity
+from agentic_security.lib import SecurityScanner
+from agentic_security.misc.banner import init_banner
 
 
 class CLI:
@@ -31,14 +32,14 @@ class CLI:
         Run Agentic Security in CI mode.
         """
         sys.path.append(os.path.dirname("."))
-        AgenticSecurity().entrypoint()
+        SecurityScanner().entrypoint()
 
     def init(self, host: str = "0.0.0.0", port: int = 8718):
         """
         Generate the default CI configuration file.
         """
         sys.path.append(os.path.dirname("."))
-        AgenticSecurity().generate_default_cfg(host, port)
+        SecurityScanner().generate_default_settings(host, port)
 
     i = init
 
@@ -47,7 +48,7 @@ class CLI:
         List all available security checks.
         """
         sys.path.append(os.path.dirname("."))
-        AgenticSecurity().list_checks()
+        SecurityScanner().list_checks()
 
 
 def main():
@@ -61,4 +62,5 @@ def main():
 
 
 if __name__ == "__main__":
+    init_banner()
     main()

agentic_security/agents/operator_crew.py

@@ -0,0 +1,254 @@
+import asyncio
+import os
+from typing import Any
+
+import httpx
+from crewai import Agent, Crew, Task
+from crewai_tools import tool
+from pydantic import BaseModel, ConfigDict, Field
+
+# Assuming LLMSpec is defined elsewhere; placeholder import
+from agentic_security.http_spec import LLMSpec
+from agentic_security.logutils import logger
+
+LLM_SPECS = []  # Populate with LLM spec strings if needed
+
+# Configure logging
+
+
+# Define AgentSpecification model
+class AgentSpecification(BaseModel):
+    name: str | None = Field(None, description="Name of the LLM/agent")
+    version: str | None = Field(None, description="Version of the LLM/agent")
+    description: str | None = Field(None, description="Description of the LLM/agent")
+    capabilities: list[str] | None = Field(None, description="List of capabilities")
+    configuration: dict[str, Any] | None = Field(
+        None, description="Configuration settings"
+    )
+    endpoint: str | None = Field(None, description="Endpoint URL of the deployed agent")
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+# Define OperatorToolBox class (unchanged from original)
+class OperatorToolBox:
+    def __init__(self, spec: AgentSpecification, datasets: list[dict[str, Any]]):
+        self.spec = spec
+        self.datasets = datasets
+        self.failures = []
+        self.llm_specs = [LLMSpec.from_string(spec) for spec in LLM_SPECS]
+
+    def get_spec(self) -> AgentSpecification:
+        return self.spec
+
+    def get_datasets(self) -> list[dict[str, Any]]:
+        return self.datasets
+
+    def validate(self) -> bool:
+        if not self.spec.name or not self.spec.version:
+            self.failures.append("Invalid specification: Name or version is missing.")
+            return False
+        if not self.datasets:
+            self.failures.append("No datasets provided.")
+            return False
+        return True
+
+    def stop(self) -> None:
+        logger.info("Stopping the toolbox...")
+
+    def run(self) -> None:
+        logger.info("Running the toolbox...")
+
+    def get_results(self) -> list[dict[str, Any]]:
+        return self.datasets
+
+    def get_failures(self) -> list[str]:
+        return self.failures
+
+    def run_operation(self, operation: str) -> str:
+        if operation not in ["dataset1", "dataset2", "dataset3"]:
+            self.failures.append(f"Operation '{operation}' failed: Dataset not found.")
+            return f"Operation '{operation}' failed: Dataset not found."
+        return f"Operation '{operation}' executed successfully."
+
+    async def test_llm_spec(self, llm_spec: LLMSpec, user_prompt: str) -> str:
+        try:
+            response = await llm_spec.verify()
+            response.raise_for_status()
+            logger.info(f"Verification succeeded for {llm_spec.url}")
+
+            test_response = await llm_spec.probe(user_prompt)
+            test_response.raise_for_status()
+            response_data = test_response.json()
+            return f"Test succeeded for {llm_spec.url}: {response_data}"
+        except httpx.HTTPStatusError as e:
+            self.failures.append(f"HTTP error occurred: {e}")
+            logger.error(f"Test failed for {llm_spec.url}: {e}")
+            return f"Test failed for {llm_spec.url}: {e}"
+        except Exception as e:
+            self.failures.append(f"An error occurred: {e}")
+            logger.error(f"Test failed for {llm_spec.url}: {e}")
+            return f"Test failed for {llm_spec.url}: {e}"
+
+    async def test_with_prompt(self, spec_index: int, user_prompt: str) -> str:
+        if not 0 <= spec_index < len(self.llm_specs):
+            return f"Invalid spec index: {spec_index}. Valid range is 0 to {len(self.llm_specs) - 1}"
+        llm_spec = self.llm_specs[spec_index]
+        return await self.test_llm_spec(llm_spec, user_prompt)
+
+
+# Define CrewAI Tools
+@tool("validate_toolbox")
+def validate_toolbox(toolbox: OperatorToolBox) -> str:
+    """Validate the toolbox configuration."""
+    is_valid = toolbox.validate()
+    return (
+        "ToolBox validation successful." if is_valid else "ToolBox validation failed."
+    )
+
+
+@tool("execute_operation")
+def execute_operation(toolbox: OperatorToolBox, operation: str) -> str:
+    """Execute a dataset operation."""
+    return toolbox.run_operation(operation)
+
+
+@tool("retrieve_results")
+def retrieve_results(toolbox: OperatorToolBox) -> str:
+    """Retrieve the results of operations."""
+    results = toolbox.get_results()
+    return (
+        f"Operation Results:\n{results}"
+        if results
+        else "No operations have been executed yet."
+    )
+
+
+@tool("retrieve_failures")
+def retrieve_failures(toolbox: OperatorToolBox) -> str:
+    """Retrieve recorded failures."""
+    failures = toolbox.get_failures()
+    return f"Failures:\n{failures}" if failures else "No failures recorded."
+
+
+@tool("list_llm_specs")
+def list_llm_specs(toolbox: OperatorToolBox) -> str:
+    """List available LLM specifications."""
+    spec_list = "\n".join(
+        f"{i}: {spec.url}" for i, spec in enumerate(toolbox.llm_specs)
+    )
+    return f"Available LLM Specs:\n{spec_list}"
+
+
+@tool("test_llm_with_prompt")
+async def test_llm_with_prompt(
+    toolbox: OperatorToolBox, spec_index: int, user_prompt: str
+) -> str:
+    """Test an LLM spec with a user prompt."""
+    return await toolbox.test_with_prompt(spec_index, user_prompt)
+
+
+# Setup OperatorToolBox
+spec = AgentSpecification(
+    name="DeepSeek Chat",
+    version="1.0",
+    description="A powerful language model",
+    capabilities=["text-generation", "question-answering"],
+    configuration={"max_tokens": 100},
+)
+toolbox = OperatorToolBox(
+    spec=spec, datasets=[{"id": "dataset1"}, {"id": "dataset2"}, {"id": "dataset3"}]
+)
+
+# Define CrewAI Agent
+dataset_manager_agent = Agent(
+    role="Dataset Manager",
+    goal="Manage and operate the OperatorToolBox to validate configurations, run operations, and test LLMs.",
+    backstory="An expert in dataset management and LLM testing, designed to assist with toolbox operations.",
+    verbose=True,
+    llm="openai",  # Using OpenAI-compatible API for DeepSeek; adjust if DeepSeek has a specific ID
+    tools=[
+        validate_toolbox,
+        execute_operation,
+        retrieve_results,
+        retrieve_failures,
+        list_llm_specs,
+        test_llm_with_prompt,
+    ],
+    allow_delegation=False,  # Single agent, no delegation needed
+)
+
+# Define Tasks
+tasks = [
+    Task(
+        description="Validate the toolbox configuration.",
+        agent=dataset_manager_agent,
+        expected_output="A string indicating whether validation succeeded or failed.",
+    ),
+    Task(
+        description="List available LLM specifications.",
+        agent=dataset_manager_agent,
+        expected_output="A string listing available LLM specs.",
+    ),
+    Task(
+        description="Guide the user to test an LLM with the prompt: 'Tell me a short story about a robot'. Suggest listing specs first.",
+        agent=dataset_manager_agent,
+        expected_output="A string suggesting the user list specs and proceed with testing.",
+    ),
+]
+
+# Define Crew
+crew = Crew(
+    agents=[dataset_manager_agent],
+    tasks=tasks,
+    verbose=2,  # Detailed logging
+)
+
+
+# Async wrapper to handle async tools
+async def run_crew():
+    # Since CrewAI's process() is synchronous but our tool is async, we need to run it in an event loop
+    result = (
+        crew.kickoff()
+    )  # Synchronous call; async tools are awaited internally by CrewAI
+    print("\nCrew Results:")
+    for task_result in result:
+        print(f"Task: {task_result.description}")
+        print(f"Output: {task_result.output}\n")
+
+    # Handle user interaction for LLM testing
+    print("Please select a spec index from the listed specs and confirm to proceed.")
+    user_input = (
+        input("Enter spec index and 'yes' to confirm (e.g., '0 yes'): ").strip().split()
+    )
+    if len(user_input) == 2 and user_input[1].lower() == "yes":
+        try:
+            spec_index = int(user_input[0])
+            user_prompt = "Tell me a short story about a robot"
+            # Create a new task for testing
+            test_task = Task(
+                description=f"Test LLM at index {spec_index} with prompt: '{user_prompt}'",
+                agent=dataset_manager_agent,
+                expected_output="A string with the test result from the LLM.",
+            )
+            test_crew = Crew(
+                agents=[dataset_manager_agent], tasks=[test_task], verbose=2
+            )
+            test_result = test_crew.kickoff()
+            print(f"Test Output: {test_result[0].output}\n")
+        except ValueError:
+            print("Invalid spec index provided.\n")
+    else:
+        print("Test canceled. Please provide a valid index and confirmation.\n")
+
+
+# Ensure DeepSeek API key is set
+os.environ["OPENAI_API_KEY"] = os.environ.get(
+    "DEEPSEEK_API_KEY", ""
+)  # CrewAI uses OPENAI_API_KEY
+os.environ[
+    "OPENAI_MODEL_NAME"
+] = "deepseek:chat"  # Specify DeepSeek model (adjust if needed)
+
+if __name__ == "__main__":
+    asyncio.run(run_crew())

agentic_security/agents/operator_pydantic.py

@@ -0,0 +1,234 @@
+import asyncio
+from typing import Any
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic_ai import Agent, RunContext, Tool
+
+# Assuming LLMSpec is defined elsewhere; placeholder import
+from agentic_security.http_spec import LLMSpec
+from agentic_security.logutils import logger
+
+LLM_SPECS = []  # Populate this list with LLM spec strings if needed
+
+
+# Define AgentSpecification model
+class AgentSpecification(BaseModel):
+    name: str | None = Field(None, description="Name of the LLM/agent")
+    version: str | None = Field(None, description="Version of the LLM/agent")
+    description: str | None = Field(None, description="Description of the LLM/agent")
+    capabilities: list[str] | None = Field(None, description="List of capabilities")
+    configuration: dict[str, Any] | None = Field(
+        None, description="Configuration settings"
+    )
+    endpoint: str | None = Field(None, description="Endpoint URL of the deployed agent")
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+# Define OperatorToolBox class
+class OperatorToolBox:
+    def __init__(self, spec: AgentSpecification, datasets: list[dict[str, Any]]):
+        self.spec = spec
+        self.datasets = datasets
+        self.failures = []
+        self.llm_specs = [LLMSpec.from_string(spec) for spec in LLM_SPECS]
+
+    def get_spec(self) -> AgentSpecification:
+        return self.spec
+
+    def get_datasets(self) -> list[dict[str, Any]]:
+        return self.datasets
+
+    def validate(self) -> bool:
+        if not self.spec.name or not self.spec.version:
+            self.failures.append("Invalid specification: Name or version is missing.")
+            return False
+        if not self.datasets:
+            self.failures.append("No datasets provided.")
+            return False
+        return True
+
+    def stop(self) -> None:
+        logger.info("Stopping the toolbox...")
+
+    def run(self) -> None:
+        logger.info("Running the toolbox...")
+
+    def get_results(self) -> list[dict[str, Any]]:
+        return self.datasets
+
+    def get_failures(self) -> list[str]:
+        return self.failures
+
+    def run_operation(self, operation: str) -> str:
+        if operation not in ["dataset1", "dataset2", "dataset3"]:
+            self.failures.append(f"Operation '{operation}' failed: Dataset not found.")
+            return f"Operation '{operation}' failed: Dataset not found."
+        return f"Operation '{operation}' executed successfully."
+
+    async def test_llm_spec(self, llm_spec: LLMSpec, user_prompt: str) -> str:
+        try:
+            response = await llm_spec.verify()
+            response.raise_for_status()
+            logger.info(f"Verification succeeded for {llm_spec.url}")
+
+            test_response = await llm_spec.probe(user_prompt)
+            test_response.raise_for_status()
+            response_data = test_response.json()
+            return f"Test succeeded for {llm_spec.url}: {response_data}"
+        except httpx.HTTPStatusError as e:
+            self.failures.append(f"HTTP error occurred: {e}")
+            logger.error(f"Test failed for {llm_spec.url}: {e}")
+            return f"Test failed for {llm_spec.url}: {e}"
+        except Exception as e:
+            self.failures.append(f"An error occurred: {e}")
+            logger.error(f"Test failed for {llm_spec.url}: {e}")
+            return f"Test failed for {llm_spec.url}: {e}"
+
+    async def test_with_prompt(self, spec_index: int, user_prompt: str) -> str:
+        if not 0 <= spec_index < len(self.llm_specs):
+            return f"Invalid spec index: {spec_index}. Valid range is 0 to {len(self.llm_specs) - 1}"
+        llm_spec = self.llm_specs[spec_index]
+        return await self.test_llm_spec(llm_spec, user_prompt)
+
+
+# Define the Agent
+class DatasetManagerAgent(Agent):
+    model: str = "deepseek:chat"
+    system_prompt: str = (
+        "You are an AI agent managing an OperatorToolBox. You can validate the toolbox, run operations, "
+        "retrieve results or failures, list LLM specs, and test LLM specs with user prompts. "
+        "Use the provided tools to assist the user based on their request."
+    )
+
+    def __init__(self, toolbox: OperatorToolBox, **kwargs):
+        super().__init__(**kwargs)
+        self.toolbox = toolbox
+
+        # Define async tools within __init__
+        async def validate_toolbox(ctx: RunContext[Any]) -> str:
+            is_valid = self.toolbox.validate()
+            return (
+                "ToolBox validation successful."
+                if is_valid
+                else "ToolBox validation failed."
+            )
+
+        async def execute_operation(ctx: RunContext[Any], operation: str) -> str:
+            return self.toolbox.run_operation(operation)
+
+        async def retrieve_results(ctx: RunContext[Any]) -> str:
+            results = self.toolbox.get_results()
+            return (
+                f"Operation Results:\n{results}"
+                if results
+                else "No operations have been executed yet."
+            )
+
+        async def retrieve_failures(ctx: RunContext[Any]) -> str:
+            failures = self.toolbox.get_failures()
+            return f"Failures:\n{failures}" if failures else "No failures recorded."
+
+        async def list_llm_specs(ctx: RunContext[Any]) -> str:
+            spec_list = "\n".join(
+                f"{i}: {spec.url}" for i, spec in enumerate(self.toolbox.llm_specs)
+            )
+            return f"Available LLM Specs:\n{spec_list}"
+
+        async def test_llm_with_prompt(
+            ctx: RunContext[Any], spec_index: int, user_prompt: str
+        ) -> str:
+            return await self.toolbox.test_with_prompt(spec_index, user_prompt)
+
+        # Register tools
+        self.tools = [
+            Tool(
+                name="validate_toolbox",
+                description="Validate the toolbox configuration.",
+                function=validate_toolbox,
+            ),
+            Tool(
+                name="execute_operation",
+                description="Execute a dataset operation.",
+                function=execute_operation,
+            ),
+            Tool(
+                name="retrieve_results",
+                description="Retrieve the results of operations.",
+                function=retrieve_results,
+            ),
+            Tool(
+                name="retrieve_failures",
+                description="Retrieve recorded failures.",
+                function=retrieve_failures,
+            ),
+            Tool(
+                name="list_llm_specs",
+                description="List available LLM specifications.",
+                function=list_llm_specs,
+            ),
+            Tool(
+                name="test_llm_with_prompt",
+                description="Test an LLM spec with a user prompt.",
+                function=test_llm_with_prompt,
+            ),
+        ]
+
+
+# Setup and run example
+async def run_dataset_manager_agent_async():
+    # Initialize OperatorToolBox with AgentSpecification
+    spec = AgentSpecification(
+        name="DeepSeek Chat",
+        version="1.0",
+        description="A powerful language model",
+        capabilities=["text-generation", "question-answering"],
+        configuration={"max_tokens": 100},
+    )
+    toolbox = OperatorToolBox(
+        spec=spec, datasets=[{"id": "dataset1"}, {"id": "dataset2"}, {"id": "dataset3"}]
+    )
+
+    # Create the agent
+    agent = DatasetManagerAgent(toolbox=toolbox)
+
+    # Example prompts
+    prompts = [
+        "Validate the toolbox.",
+        "List available LLM specs.",
+        "I want to test an LLM with my prompt: 'Tell me a short story about a robot'. Which spec index should I use?",
+    ]
+
+    for prompt in prompts:
+        result = await agent.run(prompt)
+        print(f"Prompt: {prompt}")
+        print(f"Response: {result}\n")
+
+        # Handle testing request
+        if "test an LLM with my prompt" in prompt:
+            print(
+                "Please select a spec index from the list above and confirm to proceed."
+            )
+            # Simulate user input (replace with real input in practice)
+            user_input = (
+                input("Enter spec index and 'yes' to confirm (e.g., '0 yes'): ")
+                .strip()
+                .split()
+            )
+            if len(user_input) == 2 and user_input[1].lower() == "yes":
+                try:
+                    spec_index = int(user_input[0])
+                    user_prompt = prompt.split("my prompt: ")[1].strip("'")
+                    test_result = await agent.run(
+                        f"Test LLM at index {spec_index} with prompt: {user_prompt}"
+                    )
+                    print(f"Test Response: {test_result}\n")
+                except ValueError:
+                    print("Invalid spec index provided.\n")
+            else:
+                print("Test canceled. Please provide a valid index and confirmation.\n")
+
+
+if __name__ == "__main__":
+    asyncio.run(run_dataset_manager_agent_async())

agentic_security/config.py

@@ -1,16 +1,39 @@
+from functools import lru_cache
+
 import tomli
-from loguru import logger
+
+from agentic_security.logutils import logger
+
+SETTINGS_VERSION = 1
 
 
-class CfgMixin:
+@lru_cache(maxsize=1)
+def settings_var(name: str, default=None):
+    return get_or_create_config().get_config_value(name, default)
+
+
+@lru_cache(maxsize=1)
+def get_or_create_config():
+    cfg = SettingsMixin()
+    cfg.get_or_create_config()
+    return cfg
+
+
+class SettingsMixin:
     config = {}
     default_path = "agentic_security.toml"
 
     def get_or_create_config(self) -> bool:
         if not self.has_local_config():
-            self.generate_default_cfg()
+            self.generate_default_settings()
             return False
         self.load_config(self.default_path)
+        settings_version = self.get_config_value("general.version")
+        if settings_version and settings_version != SETTINGS_VERSION:
+            logger.error(
+                f"Configuration version mismatch: expected {SETTINGS_VERSION}, got {settings_version}."
+            )
+            return False
         return True
 
     def has_local_config(self):
@@ -64,7 +87,7 @@ class CfgMixin:
                 return default
         return value
 
-    def generate_default_cfg(self, host: str = "0.0.0.0", port: int = 8718):
+    def generate_default_settings(self, host: str = "0.0.0.0", port: int = 8718):
         # Accept host / port as parameters
         with open(self.default_path, "w") as f:
             f.write(
@@ -84,6 +107,7 @@ maxBudget = 1000000 # Maximum budget for the scan
 max_th = 0.3 # Maximum failure threshold (percentage)
 optimize = false # Enable optimization during scanning
 enableMultiStepAttack = false # Enable multi-step attack simulations
+version = $SETTINGS_VERSION
 
 # [modules.LLM-Jailbreak-Classifier]
 # dataset_name = "markush1/LLM-Jailbreak-Classifier"
@@ -110,11 +134,20 @@ high = 0.5
 OPENAI_API_KEY = "$OPENAI_API_KEY"
 DEEPSEEK_API_KEY = "$DEEPSEEK_API_KEY"
 
+[caching]
+enable = true
+cache_size = 10000
+use_disk_cache = false
+
+[network]
+retry = 3
+timeout_connect = 30
+timeout_response = 90
 """.replace(
                     "$HOST", host
-                ).replace(
-                    "$PORT", str(port)
                 )
+                .replace("$PORT", str(port))
+                .replace("$SETTINGS_VERSION", str(SETTINGS_VERSION))
             )
 
         logger.info(

agentic_security/core/app.py

@@ -2,16 +2,21 @@ import os
 from asyncio import Event, Queue
 
 from fastapi import FastAPI
+from fastapi.responses import ORJSONResponse
+
+from agentic_security.http_spec import LLMSpec
 
 tools_inbox: Queue = Queue()
 stop_event: Event = Event()
 current_run: str = {"spec": "", "id": ""}
-_secrets = {}
+_secrets: dict[str, str] = {}
+
+current_run: dict[str, int | LLMSpec] = {"spec": "", "id": ""}
 
 
 def create_app() -> FastAPI:
     """Create and configure the FastAPI application."""
-    app = FastAPI()
+    app = FastAPI(default_response_class=ORJSONResponse)
     return app
 
 
@@ -25,29 +30,29 @@ def get_stop_event() -> Event:
     return stop_event
 
 
-def get_current_run() -> str:
+def get_current_run() -> dict[str, int | LLMSpec]:
     """Get the current run id."""
     return current_run
 
 
-def set_current_run(spec):
+def set_current_run(spec: LLMSpec) -> dict[str, int | LLMSpec]:
     """Set the current run id."""
     current_run["id"] = hash(id(spec))
     current_run["spec"] = spec
     return current_run
 
 
-def get_secrets():
+def get_secrets() -> dict[str, str]:
     return _secrets
 
 
-def set_secrets(secrets):
+def set_secrets(secrets: dict[str, str]) -> dict[str, str]:
     _secrets.update(secrets)
     expand_secrets(_secrets)
     return _secrets
 
 
-def expand_secrets(secrets):
+def expand_secrets(secrets: dict[str, str]) -> None:
     for key in secrets:
         val = secrets[key]
         if val.startswith("$"):

agentic_security/core/logging.py

@@ -1,26 +1,5 @@
-from logging import config
+from agentic_security.logutils import set_log_level_to_info
 
 
 def setup_logging():
-    config.dictConfig(
-        {
-            "version": 1,
-            "disable_existing_loggers": True,
-            "handlers": {
-                "console": {
-                    "class": "logging.StreamHandler",
-                },
-            },
-            "root": {
-                "handlers": ["console"],
-                "level": "INFO",
-            },
-            "loggers": {
-                "uvicorn.access": {
-                    "level": "ERROR",  # Set higher log level to suppress info logs globally
-                    "handlers": ["console"],
-                    "propagate": False,
-                }
-            },
-        }
-    )
+    return set_log_level_to_info()

agentic_security/dependencies.py

@@ -1,13 +1,11 @@
-from agentic_security.config import CfgMixin
+from agentic_security.config import get_or_create_config
 from agentic_security.core.app import set_secrets
 
 
 class InMemorySecrets:
     def __init__(self):
-        self.secrets = {}
-        self.config = CfgMixin()
-        self.config.get_or_create_config()
-        self.secrets = self.config.config.get("secrets", {})
+        config = get_or_create_config()
+        self.secrets = config.get_config_value("secrets", {})
         set_secrets(self.secrets)
 
     def set_secret(self, key: str, value: str):

agentic_security/http_spec.py

@@ -1,9 +1,12 @@
 import base64
 from enum import Enum
+from urllib.parse import urlparse
 
 import httpx
 from pydantic import BaseModel
 
+from agentic_security.config import settings_var
+
 
 class Modality(Enum):
     TEXT = 0
@@ -28,7 +31,7 @@ def encode_audio_base64_by_url(url: str) -> str:
 
 
 class InvalidHTTPSpecError(Exception):
-    ...
+    pass
 
 
 class LLMSpec(BaseModel):
@@ -47,14 +50,21 @@ class LLMSpec(BaseModel):
         except Exception as e:
             raise InvalidHTTPSpecError(f"Failed to parse HTTP spec: {e}") from e
 
+    def timeout(self):
+        return (
+            settings_var("network.timeout_connect", 30),
+            settings_var("network.timeout_response", 90),
+        )
+
     async def _probe_with_files(self, files):
-        async with httpx.AsyncClient() as client:
+        transport = httpx.AsyncHTTPTransport(retries=settings_var("network.retry", 3))
+        async with httpx.AsyncClient(transport=transport) as client:
             response = await client.request(
                 method=self.method,
                 url=self.url,
                 headers=self.headers,
                 files=files,
-                timeout=(30, 90),
+                timeout=self.timeout(),
             )
 
         return response
@@ -90,13 +100,15 @@ class LLMSpec(BaseModel):
         content = self.body.replace("<<PROMPT>>", escape_special_chars_for_json(prompt))
         content = content.replace("<<BASE64_IMAGE>>", encoded_image)
         content = content.replace("<<BASE64_AUDIO>>", encoded_audio)
-        async with httpx.AsyncClient() as client:
+
+        transport = httpx.AsyncHTTPTransport(retries=settings_var("network.retry", 3))
+        async with httpx.AsyncClient(transport=transport) as client:
             response = await client.request(
                 method=self.method,
                 url=self.url,
                 headers=self.headers,
                 content=content,
-                timeout=(30, 90),
+                timeout=self.timeout(),
             )
 
         return response
@@ -148,6 +160,14 @@ def parse_http_spec(http_spec: str) -> LLMSpec:
     # Extract the method and URL from the first line
     method, url = lines[0].split(" ")[0:2]
 
+    # Check url validity
+    valid_url = urlparse(url)
+    # if missing the correct formatting ://, urlparse.netloc will be empty
+    if valid_url.scheme not in ("http", "https") or not valid_url.netloc:
+        raise InvalidHTTPSpecError(
+            f"Invalid URL: {url}. Ensure it starts with 'http://' or 'https://'"
+        )
+
     # Initialize headers and body
     headers = {}
     body = ""

agentic_security/lib.py

@@ -4,13 +4,13 @@ from datetime import datetime
 
 import colorama
 import tqdm.asyncio
-from loguru import logger
 from rich.console import Console
 from rich.table import Table
 from tabulate import tabulate
 
-from agentic_security.config import CfgMixin  # Importing the configuration mixin
-from agentic_security.models.schemas import Scan
+from agentic_security.config import SettingsMixin  # Importing the configuration mixin
+from agentic_security.logutils import logger
+from agentic_security.primitives import Scan
 from agentic_security.probe_data import REGISTRY
 from agentic_security.routes.scan import streaming_response_generator
 
@@ -23,7 +23,7 @@ YELLOW = colorama.Fore.YELLOW
 BLUE = colorama.Fore.BLUE
 
 
-class AgenticSecurity(CfgMixin):
+class SecurityScanner(SettingsMixin):
     @classmethod
     async def async_scan(
         cls,
@@ -202,7 +202,7 @@ class AgenticSecurity(CfgMixin):
         self.load_config(self.default_path)
         logger.info("Configuration loaded successfully.")
         print(self.config)
-        datasets = list(self.get_config_value("modules").values())
+        datasets = list(self.get_config_value("modules", {}).values())
         for d in datasets:
             d["selected"] = True
         self.scan(

agentic_security/logutils.py

@@ -0,0 +1,85 @@
+# import sys
+
+# from loguru import logger
+
+# # Define custom colors
+# BLUE = "#89CFF0"
+# BROWN = "#8B4513"  # Brown for DEBUG
+
+# # Define custom log level colors
+# logger.level("DEBUG", color=f"<fg {BROWN}>")
+# logger.level("INFO", color=f"<fg {BLUE}>")
+
+# # Define custom log format with aligned messages and colored levels
+# LOG_FORMAT = (
+#     "<level>{level:<8}</level> "  # Properly formatted and colored log level
+#     "<level>{message:<100}</level> "  # Left-aligned message for readability
+#     "<cyan>{file.name}</cyan>:<cyan>{line}</cyan>"  # File name and line number in cyan
+# )
+
+# # Remove default handlers and add a new one with custom formatting
+# logger.remove()
+# logger.add(sys.stdout, format=LOG_FORMAT, level="DEBUG", colorize=True)
+import logging
+import logging.config
+from os import getenv
+
+LOGGER_NAME = None
+
+LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "rich": {"format": "%(message)s", "datefmt": "[%X]"},
+    },
+    "handlers": {
+        "rich": {
+            "class": "rich.logging.RichHandler",
+            "level": "INFO",
+            "formatter": "rich",
+            "show_time": False,
+            "rich_tracebacks": False,
+            "show_path": lambda: True if getenv("API_RUNTIME") == "dev" else False,
+            "tracebacks_show_locals": False,
+        },
+    },
+    "loggers": {
+        "": {  # Root logger configuration
+            "level": "INFO",
+            "handlers": ["rich"],
+            "propagate": True,
+        },
+    },
+}
+
+
+def configure_logging():
+    # Apply the dictionary configuration
+    logging.config.dictConfig(LOGGING_CONFIG)
+
+    # Get and return the logger
+    logger = logging.getLogger(LOGGER_NAME)
+    return logger
+
+
+logger: logging.Logger = configure_logging()
+
+
+def set_log_level_to_debug():
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(logging.DEBUG)
+    # Update handler level as well
+    for handler in logger.handlers:
+        handler.setLevel(logging.DEBUG)
+
+
+def set_log_level_to_info():
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(logging.INFO)
+    # Update handler level as well
+    for handler in logger.handlers:
+        handler.setLevel(logging.INFO)
+
+
+# Set initial log level
+set_log_level_to_info()

agentic_security/middleware/logging.py

@@ -1,7 +1,8 @@
 from fastapi import Request
-from loguru import logger
 from starlette.middleware.base import BaseHTTPMiddleware
 
+from agentic_security.logutils import logger
+
 
 class LogNon200ResponsesMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next):

agentic_security/misc/banner.py

@@ -0,0 +1,93 @@
+from pyfiglet import Figlet, FontNotFound
+from termcolor import colored
+
+try:
+    from importlib.metadata import version
+except ImportError:
+    from importlib_metadata import version
+
+
+def generate_banner(
+    title: str = "Agentic Security",
+    font: str = "slant",
+    version: str = "v2.1.0",
+    tagline: str = "Proactive Threat Detection & Automated Security Protocols",
+    author: str = "Developed by: [Security Team]",
+    website: str = "Website: https://github.com/msoedov/agentic_security",
+    warning: str | None = "",  # Using Optional for warning since it might be None
+) -> str:
+    """Generate a visually enhanced banner with dynamic width and borders."""
+    # Define the text elements
+
+    # Initialize Figlet with the specified font, fallback to default if not found
+    try:
+        f = Figlet(font=font)
+    except FontNotFound:
+        f = Figlet()  # Fallback to default font
+
+    # Render the title text and calculate the maximum width of Figlet lines
+    banner_text = f.renderText(title)
+    banner_lines = banner_text.splitlines()
+    figlet_max_width = max(len(line) for line in banner_lines) if banner_lines else 0
+
+    # Create the details line and calculate its width
+    details_line = f"Version: {version} | {website}"
+    details_width = len(details_line)
+
+    # Calculate widths of other text elements
+    warning_width = len(warning)
+    tagline_width = len(tagline)
+
+    # Determine the overall maximum width for centering
+    overall_max_width = max(
+        figlet_max_width, warning_width, tagline_width, details_width
+    )
+
+    # Pad the Figlet lines to the overall maximum width
+    padded_banner_lines = [line.center(overall_max_width) for line in banner_lines]
+
+    # Define decorative characters and colors
+    decor_chars = ["▄", "■", "►"]
+    decor_colors = ["blue", "red", "yellow"]
+
+    # Create and color the content lines
+    content_lines = []
+    for line in padded_banner_lines:
+        content_lines.append(colored(line, "blue"))
+    content_lines.append(colored(decor_chars[0] * overall_max_width, decor_colors[0]))
+    content_lines.append(
+        colored(warning.center(overall_max_width), "red", attrs=["blink", "bold"])
+    )
+    content_lines.append(colored(decor_chars[1] * overall_max_width, decor_colors[1]))
+    content_lines.append(colored(tagline.center(overall_max_width), "red"))
+    content_lines.append(colored(decor_chars[2] * overall_max_width, decor_colors[2]))
+    content_lines.append(colored(details_line.center(overall_max_width), "magenta"))
+
+    # Define border color and create top and bottom borders
+    border_color = "blue"
+    top_border = colored("╔" + "═" * (overall_max_width + 2) + "╗", border_color)
+    bottom_border = colored("╚" + "═" * (overall_max_width + 2) + "╝", border_color)
+
+    # Add side borders to each content line with padding
+    bordered_content = [
+        colored("║ ", border_color) + line + colored(" ║", border_color)
+        for line in content_lines
+    ]
+
+    # Assemble the full banner
+    banner = top_border + "\n" + "\n".join(bordered_content) + "\n" + bottom_border
+    return banner
+
+
+def init_banner():
+    return
+    ver = version("agentic_security")
+    try:
+        print(generate_banner(version=ver))
+    except Exception:
+        # UnicodeEncodeError with codec on some systems
+        pass
+
+
+if __name__ == "__main__":
+    init_banner()

agentic_security/primitives/__init__.py

@@ -0,0 +1,11 @@
+from agentic_security.primitives.models import (  # noqa
+    CompletionRequest,
+    FileProbeResponse,
+    LLMInfo,
+    Message,
+    Probe,
+    Scan,
+    ScanResult,
+    Settings,
+    Table,
+)

agentic_security/probe_actor/fuzzer.py

@@ -2,15 +2,16 @@ import asyncio
 import random
 import time
 from collections.abc import AsyncGenerator
+from json import JSONDecodeError
 
 import httpx
 import pandas as pd
-from loguru import logger
 from skopt import Optimizer
 from skopt.space import Real
 
 from agentic_security.http_spec import Modality
-from agentic_security.models.schemas import Scan, ScanResult
+from agentic_security.logutils import logger
+from agentic_security.primitives import Scan, ScanResult
 from agentic_security.probe_actor.cost_module import calculate_cost
 from agentic_security.probe_actor.refusal import refusal_heuristic
 from agentic_security.probe_data import audio_generator, image_generator, msj_data
@@ -19,6 +20,10 @@ from agentic_security.probe_data.data import prepare_prompts
 # TODO: full log file
 
 MAX_PROMPT_LENGTH = 2048
+BUDGET_MULTIPLIER = 100_000_000
+INITIAL_OPTIMIZER_POINTS = 25
+MIN_FAILURE_SAMPLES = 5
+FAILURE_RATE_THRESHOLD = 0.5
 
 
 async def generate_prompts(
@@ -75,6 +80,42 @@ async def process_prompt(
         logger.error(f"Request error: {exc}")
         errors.append((module_name, prompt, "?", str(exc)))
         return tokens, True
+    except JSONDecodeError as json_decode_error:
+        logger.error(f"Jason error: {json_decode_error}")
+        errors.append((module_name, prompt, "?", str(json_decode_error)))
+        return tokens, True
+
+
+async def process_prompt_batch(
+    request_factory,
+    prompts: list[str],
+    tokens: int,
+    module_name: str,
+    refusals,
+    errors,
+    outputs,
+) -> tuple[int, int]:
+    tasks = [
+        process_prompt(
+            request_factory, p, tokens, module_name, refusals, errors, outputs
+        )
+        for p in prompts
+    ]
+    results = await asyncio.gather(*tasks)
+    total_tokens = sum(r[0] for r in results)
+    failures = sum(1 for r in results if r[1])
+    return total_tokens, failures
+
+
+async def with_error_handling(agen):
+    try:
+        async for t in agen:
+            yield t
+    except Exception as e:
+        logger.exception("Scan failed")
+        yield ScanResult.status_msg(f"Scan failed: {str(e)}")
+    finally:
+        yield ScanResult.status_msg("Scan completed.")
 
 
 async def perform_single_shot_scan(
@@ -87,126 +128,120 @@ async def perform_single_shot_scan(
     secrets: dict[str, str] = {},
 ) -> AsyncGenerator[str, None]:
     """Perform a standard security scan."""
-    max_budget = max_budget * 100_000_000
+    max_budget = max_budget * BUDGET_MULTIPLIER
     selected_datasets = [m for m in datasets if m["selected"]]
     request_factory = multi_modality_spec(request_factory)
-    try:
-        yield ScanResult.status_msg("Loading datasets...")
-        prompt_modules = prepare_prompts(
-            dataset_names=[m["dataset_name"] for m in selected_datasets],
-            budget=max_budget,
-            tools_inbox=tools_inbox,
-            options=[m.get("opts", {}) for m in selected_datasets],
-        )
-        yield ScanResult.status_msg("Datasets loaded. Starting scan...")
+    yield ScanResult.status_msg("Loading datasets...")
+    prompt_modules = prepare_prompts(
+        dataset_names=[m["dataset_name"] for m in selected_datasets],
+        budget=max_budget,
+        tools_inbox=tools_inbox,
+        options=[m.get("opts", {}) for m in selected_datasets],
+    )
+    yield ScanResult.status_msg("Datasets loaded. Starting scan...")
 
-        errors = []
-        refusals = []
-        outputs = []
-        total_prompts = sum(len(m.prompts) for m in prompt_modules if not m.lazy)
-        processed_prompts = 0
+    errors = []
+    refusals = []
+    outputs = []
+    total_prompts = sum(len(m.prompts) for m in prompt_modules if not m.lazy)
+    processed_prompts = 0
 
-        optimizer = (
-            Optimizer([Real(0, 1)], base_estimator="GP", n_initial_points=25)
-            if optimize
-            else None
-        )
-        failure_rates = []
+    optimizer = (
+        Optimizer([Real(0, 1)], base_estimator="GP", n_initial_points=25)
+        if optimize
+        else None
+    )
+    failure_rates = []
 
-        total_tokens = 0
+    total_tokens = 0
+    tokens = 0
+    should_stop = False
+    for module in prompt_modules:
+        if should_stop:
+            break
         tokens = 0
-        should_stop = False
-        for module in prompt_modules:
-            if should_stop:
-                break
-            tokens = 0
-            module_failures = 0
-            module_size = 0 if module.lazy else len(module.prompts)
-            logger.info(f"Scanning {module.dataset_name} {module_size}")
+        module_failures = 0
+        module_size = 0 if module.lazy else len(module.prompts)
+        logger.info(f"Scanning {module.dataset_name} {module_size}")
+        module_prompts = 0  # Reset for each module
 
-            async for prompt in generate_prompts(module.prompts):
-                if stop_event and stop_event.is_set():
-                    stop_event.clear()
-                    logger.info("Scan stopped by user.")
-                    yield ScanResult.status_msg("Scan stopped by user.")
-                    return
+        async for prompt in generate_prompts(module.prompts):
+            if stop_event and stop_event.is_set():
+                stop_event.clear()
+                logger.info("Scan stopped by user.")
+                yield ScanResult.status_msg("Scan stopped by user.")
+                return
 
-                processed_prompts += 1
-                progress = (
-                    100 * processed_prompts / total_prompts if total_prompts else 0
-                )
-                total_tokens -= tokens
-                start = time.time()
-                tokens, failed = await process_prompt(
-                    request_factory,
-                    prompt,
-                    tokens,
-                    module.dataset_name,
-                    refusals,
-                    errors,
-                    outputs,
-                )
-                end = time.time()
-                total_tokens += tokens
-                # logger.debug(f"Trying prompt: {prompt}, {failed=}")
-                if failed:
-                    module_failures += 1
-                failure_rate = module_failures / max(processed_prompts, 1)
-                failure_rates.append(failure_rate)
-                cost = calculate_cost(tokens)
+            processed_prompts += 1
+            module_prompts += 1  # Fixed increment syntax
+            # Calculate progress based on total processed prompts
+            progress = 100 * processed_prompts / total_prompts if total_prompts else 0
+            progress = progress % 100
 
-                # TODO: improve this cond
-                last_output = outputs[-1] if outputs else None
-                if last_output and last_output[1] == prompt:
-                    response_text = last_output[2]
-                else:
-                    response_text = ""
+            total_tokens -= tokens
+            start = time.time()
+            tokens, failed = await process_prompt(
+                request_factory,
+                prompt,
+                tokens,
+                module.dataset_name,
+                refusals,
+                errors,
+                outputs,
+            )
+            end = time.time()
+            total_tokens += tokens
 
-                yield ScanResult(
-                    module=module.dataset_name,
-                    tokens=round(tokens / 1000, 1),
-                    cost=cost,
-                    progress=round(progress, 2),
-                    failureRate=round(failure_rate * 100, 2),
-                    prompt=prompt[:MAX_PROMPT_LENGTH],
-                    latency=end - start,
-                    model=response_text,
-                ).model_dump_json()
+            if failed:
+                module_failures += 1
+            failure_rate = module_failures / max(module_prompts, 1)
+            failure_rates.append(failure_rate)
+            cost = calculate_cost(tokens)
 
-                if optimize and len(failure_rates) >= 5:
-                    next_point = optimizer.ask()
-                    optimizer.tell(next_point, -failure_rate)
-                    best_failure_rate = -optimizer.get_result().fun
-                    if best_failure_rate > 0.5:
-                        yield ScanResult.status_msg(
-                            f"High failure rate detected ({best_failure_rate:.2%}). Stopping this module..."
-                        )
-                        should_stop = True
-                        break
-                if total_tokens > max_budget:
-                    logger.info(
-                        f"Scan ran out of budget and stopped. {total_tokens=} {max_budget=}"
-                    )
+            last_output = outputs[-1] if outputs else None
+            if last_output and last_output[1] == prompt:
+                response_text = last_output[2]
+            else:
+                response_text = ""
+
+            yield ScanResult(
+                module=module.dataset_name,
+                tokens=round(tokens / 1000, 1),
+                cost=cost,
+                progress=round(progress, 2),
+                failureRate=round(failure_rate * 100, 2),
+                prompt=prompt[:MAX_PROMPT_LENGTH],
+                latency=end - start,
+                model=response_text,
+            ).model_dump_json()
+
+            if optimize and len(failure_rates) >= 5:
+                next_point = optimizer.ask()
+                optimizer.tell(next_point, -failure_rate)
+                best_failure_rate = -optimizer.get_result().fun
+                if best_failure_rate > 0.5:
                     yield ScanResult.status_msg(
-                        f"Scan ran out of budget and stopped. {total_tokens=} {max_budget=}"
+                        f"High failure rate detected ({best_failure_rate:.2%}). Stopping this module..."
                     )
                     should_stop = True
                     break
+            if total_tokens > max_budget:
+                logger.info(
+                    f"Scan ran out of budget and stopped. {total_tokens=} {max_budget=}"
+                )
+                yield ScanResult.status_msg(
+                    f"Scan ran out of budget and stopped. {total_tokens=} {max_budget=}"
+                )
+                should_stop = True
+                break
 
-        yield ScanResult.status_msg("Scan completed.")
+    yield ScanResult.status_msg("Scan completed.")
 
-        failure_data = errors + refusals
-        df = pd.DataFrame(
-            failure_data, columns=["module", "prompt", "status_code", "content"]
-        )
-        df.to_csv("failures.csv", index=False)
-
-    except Exception as e:
-        logger.exception("Scan failed")
-        yield ScanResult.status_msg(f"Scan failed: {str(e)}")
-        # raise e
-    finally:
-        yield ScanResult.status_msg("Scan completed.")
+    failure_data = errors + refusals
+    df = pd.DataFrame(
+        failure_data, columns=["module", "prompt", "status_code", "content"]
+    )
+    df.to_csv("failures.csv", index=False)
 
 
 async def perform_many_shot_scan(
@@ -223,114 +258,107 @@ async def perform_many_shot_scan(
 ) -> AsyncGenerator[str, None]:
     """Perform a multi-step security scan with probe injection."""
     request_factory = multi_modality_spec(request_factory)
-    try:
-        # Load main and probe datasets
-        yield ScanResult.status_msg("Loading datasets...")
-        prompt_modules = prepare_prompts(
-            dataset_names=[m["dataset_name"] for m in datasets if m["selected"]],
-            budget=max_budget,
-            tools_inbox=tools_inbox,
-        )
-        yield ScanResult.status_msg("Loading datasets for MSJ...")
-        msj_modules = msj_data.prepare_prompts(probe_datasets)
-        yield ScanResult.status_msg("Datasets loaded. Starting scan...")
+    # Load main and probe datasets
+    yield ScanResult.status_msg("Loading datasets...")
+    prompt_modules = prepare_prompts(
+        dataset_names=[m["dataset_name"] for m in datasets if m["selected"]],
+        budget=max_budget,
+        tools_inbox=tools_inbox,
+    )
+    yield ScanResult.status_msg("Loading datasets for MSJ...")
+    msj_modules = msj_data.prepare_prompts(probe_datasets)
+    yield ScanResult.status_msg("Datasets loaded. Starting scan...")
 
-        errors = []
-        refusals = []
-        outputs = []
-        total_prompts = sum(len(m.prompts) for m in prompt_modules if not m.lazy)
-        processed_prompts = 0
+    errors = []
+    refusals = []
+    outputs = []
+    total_prompts = sum(len(m.prompts) for m in prompt_modules if not m.lazy)
+    processed_prompts = 0
 
-        optimizer = (
-            Optimizer([Real(0, 1)], base_estimator="GP", n_initial_points=25)
-            if optimize
-            else None
-        )
-        failure_rates = []
+    optimizer = (
+        Optimizer([Real(0, 1)], base_estimator="GP", n_initial_points=25)
+        if optimize
+        else None
+    )
+    failure_rates = []
 
-        for module in prompt_modules:
-            module_failures = 0
-            module_size = 0 if module.lazy else len(module.prompts)
-            logger.info(f"Scanning {module.dataset_name} {module_size}")
+    for module in prompt_modules:
+        module_failures = 0
+        module_size = 0 if module.lazy else len(module.prompts)
+        logger.info(f"Scanning {module.dataset_name} {module_size}")
 
-            async for prompt in generate_prompts(module.prompts):
-                if stop_event and stop_event.is_set():
-                    stop_event.clear()
-                    logger.info("Scan stopped by user.")
-                    yield ScanResult.status_msg("Scan stopped by user.")
-                    return
-                tokens = 0
-                processed_prompts += 1
-                progress = (
-                    100 * processed_prompts / total_prompts if total_prompts else 0
+        async for prompt in generate_prompts(module.prompts):
+            if stop_event and stop_event.is_set():
+                stop_event.clear()
+                logger.info("Scan stopped by user.")
+                yield ScanResult.status_msg("Scan stopped by user.")
+                return
+            tokens = 0
+            processed_prompts += 1
+            progress = 100 * processed_prompts / total_prompts if total_prompts else 0
+            progress = progress % 100
+
+            full_prompt = ""
+            msj_module = random.choice(msj_modules)
+
+            prompt_tokens = len(full_prompt.split())
+            tokens += prompt_tokens
+
+            injected = False
+            for _ in range(20):
+                if injected:
+                    break
+
+                m_prompt = random.choice(msj_module.prompts)
+                full_prompt += "\n" + m_prompt
+                if tokens > max_ctx_length:
+                    full_prompt = "\n" + prompt
+                    injected = True
+
+                tokens, failed = await process_prompt(
+                    request_factory,
+                    full_prompt,
+                    tokens,
+                    module.dataset_name,
+                    refusals,
+                    errors,
+                    outputs,
                 )
+                if failed:
+                    module_failures += 1
+                    break
+                if injected:
+                    break
 
-                full_prompt = ""
-                msj_module = random.choice(msj_modules)
+            failure_rate = module_failures / max(processed_prompts, 1)
+            failure_rates.append(failure_rate)
+            cost = calculate_cost(tokens)
 
-                prompt_tokens = len(full_prompt.split())
-                tokens += prompt_tokens
+            yield ScanResult(
+                module=module.dataset_name,
+                tokens=round(tokens / 1000, 1),
+                cost=cost,
+                progress=round(progress, 2),
+                failureRate=round(failure_rate * 100, 2),
+                prompt=prompt[:MAX_PROMPT_LENGTH],
+            ).model_dump_json()
 
-                injected = False
-                for _ in range(20):
-                    if injected:
-                        break
-
-                    m_prompt = random.choice(msj_module.prompts)
-                    full_prompt += "\n" + m_prompt
-                    if tokens > max_ctx_length:
-                        full_prompt = "\n" + prompt
-                        injected = True
-
-                    tokens, failed = await process_prompt(
-                        request_factory,
-                        full_prompt,
-                        tokens,
-                        module.dataset_name,
-                        refusals,
-                        errors,
-                        outputs,
+            if optimize and len(failure_rates) >= 5:
+                next_point = optimizer.ask()
+                optimizer.tell(next_point, -failure_rate)
+                best_failure_rate = -optimizer.get_result().fun
+                if best_failure_rate > 0.5:
+                    yield ScanResult.status_msg(
+                        f"High failure rate detected ({best_failure_rate:.2%}). Stopping this module..."
                     )
-                    if failed:
-                        module_failures += 1
-                        break
-                    if injected:
-                        break
+                    break
 
-                failure_rate = module_failures / max(processed_prompts, 1)
-                failure_rates.append(failure_rate)
-                cost = calculate_cost(tokens)
+    yield ScanResult.status_msg("Scan completed.")
 
-                yield ScanResult(
-                    module=module.dataset_name,
-                    tokens=round(tokens / 1000, 1),
-                    cost=cost,
-                    progress=round(progress, 2),
-                    failureRate=round(failure_rate * 100, 2),
-                    prompt=prompt[:MAX_PROMPT_LENGTH],
-                ).model_dump_json()
-
-                if optimize and len(failure_rates) >= 5:
-                    next_point = optimizer.ask()
-                    optimizer.tell(next_point, -failure_rate)
-                    best_failure_rate = -optimizer.get_result().fun
-                    if best_failure_rate > 0.5:
-                        yield ScanResult.status_msg(
-                            f"High failure rate detected ({best_failure_rate:.2%}). Stopping this module..."
-                        )
-                        break
-
-        yield ScanResult.status_msg("Scan completed.")
-
-        df = pd.DataFrame(
-            errors + refusals, columns=["module", "prompt", "status_code", "content"]
-        )
-        df.to_csv("failures.csv", index=False)
-
-    except Exception as e:
-        logger.exception("Scan failed")
-        yield ScanResult.status_msg(f"Scan failed: {str(e)}")
-        raise e
+    df = pd.DataFrame(
+        errors + refusals, columns=["module", "prompt", "status_code", "content"]
+    )
+    df.to_csv("failures.csv", index=False)
 
 
 def scan_router(
@@ -340,23 +368,27 @@ def scan_router(
     stop_event: asyncio.Event = None,
 ):
     if scan_parameters.enableMultiStepAttack:
-        return perform_many_shot_scan(
-            request_factory=request_factory,
-            max_budget=scan_parameters.maxBudget,
-            datasets=scan_parameters.datasets,
-            probe_datasets=scan_parameters.probe_datasets,
-            tools_inbox=tools_inbox,
-            optimize=scan_parameters.optimize,
-            stop_event=stop_event,
-            secrets=scan_parameters.secrets,
+        return with_error_handling(
+            perform_many_shot_scan(
+                request_factory=request_factory,
+                max_budget=scan_parameters.maxBudget,
+                datasets=scan_parameters.datasets,
+                probe_datasets=scan_parameters.probe_datasets,
+                tools_inbox=tools_inbox,
+                optimize=scan_parameters.optimize,
+                stop_event=stop_event,
+                secrets=scan_parameters.secrets,
+            )
         )
     else:
-        return perform_single_shot_scan(
-            request_factory=request_factory,
-            max_budget=scan_parameters.maxBudget,
-            datasets=scan_parameters.datasets,
-            tools_inbox=tools_inbox,
-            optimize=scan_parameters.optimize,
-            stop_event=stop_event,
-            secrets=scan_parameters.secrets,
+        return with_error_handling(
+            perform_single_shot_scan(
+                request_factory=request_factory,
+                max_budget=scan_parameters.maxBudget,
+                datasets=scan_parameters.datasets,
+                tools_inbox=tools_inbox,
+                optimize=scan_parameters.optimize,
+                stop_event=stop_event,
+                secrets=scan_parameters.secrets,
+            )
         )

agentic_security/probe_actor/operator.py

@@ -1,15 +1,14 @@
 import asyncio
-import logging
 from typing import Any
 
 import httpx
-from httpx import LLMSpec
 from pydantic import BaseModel, Field
 from pydantic_ai import Agent, RunContext
 
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
+from agentic_security.http_spec import LLMSpec
+from agentic_security.logutils import logger
+
+LLM_SPECS = []
 
 
 class AgentSpecification(BaseModel):
@@ -28,6 +27,7 @@ class OperatorToolBox:
         self.spec = spec
         self.datasets = datasets
         self.failures = []
+        self.llm_specs = [LLMSpec.from_string(spec) for spec in LLM_SPECS]
 
     def get_spec(self) -> AgentSpecification:
         return self.spec
@@ -62,52 +62,33 @@ class OperatorToolBox:
             return f"Operation '{operation}' failed: Dataset not found."
         return f"Operation '{operation}' executed successfully."
 
-    async def test(self, description: str, sample_test: dict[str, Any]) -> str:
-        agent = Agent(
-            "openai:gpt-4o",
-            result_type=LLMSpec,
-            system_prompt="Extract the LLM specification from the input",
-        )
+    async def test_llm_spec(self, llm_spec: LLMSpec, user_prompt: str) -> str:
+        try:
+            # Verify the spec
+            response = await llm_spec.verify()
+            response.raise_for_status()
+            logger.info(f"Verification succeeded for {llm_spec.url}")
 
-        async with agent.run_stream(description) as result:
-            async for spec in result.stream():
-                self.spec.endpoint = spec.url
+            # Run test with user prompt
+            test_response = await llm_spec.probe(user_prompt)
+            test_response.raise_for_status()
+            response_data = test_response.json()
+            return f"Test succeeded for {llm_spec.url}: {response_data}"
+        except httpx.HTTPStatusError as e:
+            self.failures.append(f"HTTP error occurred: {e}")
+            logger.error(f"Test failed for {llm_spec.url}: {e}")
+            return f"Test failed for {llm_spec.url}: {e}"
+        except Exception as e:
+            self.failures.append(f"An error occurred: {e}")
+            logger.error(f"Test failed for {llm_spec.url}: {e}")
+            return f"Test failed for {llm_spec.url}: {e}"
 
-                # Verify access to the endpoint
-                async with httpx.AsyncClient() as client:
-                    try:
-                        access_response = await client.get(spec.url)
-                        access_response.raise_for_status()
-                    except httpx.HTTPStatusError as e:
-                        self.failures.append(f"HTTP error occurred: {e}")
-                        logger.error(f"Access verification failed: {e}")
-                        return f"Access verification failed: {e}"
-                    except Exception as e:
-                        self.failures.append(f"An error occurred: {e}")
-                        logger.error(f"Access verification failed: {e}")
-                        return f"Access verification failed: {e}"
+    async def test_with_prompt(self, spec_index: int, user_prompt: str) -> str:
+        if not 0 <= spec_index < len(self.llm_specs):
+            return f"Invalid spec index: {spec_index}. Valid range is 0 to {len(self.llm_specs) - 1}"
 
-                # Run the sample test
-                try:
-                    test_response = await client.post(
-                        f"{spec.url}/test", json=sample_test
-                    )
-                    test_response.raise_for_status()
-                    response_data = test_response.json()
-                    if "choices" in response_data and len(response_data["choices"]) > 0:
-                        return f"Testing agent at {spec.url} succeeded: {response_data}"
-                    else:
-                        self.failures.append("Invalid response format")
-                        logger.error("Sample test failed: Invalid response format")
-                        return "Sample test failed: Invalid response format"
-                except httpx.HTTPStatusError as e:
-                    self.failures.append(f"HTTP error occurred: {e}")
-                    logger.error(f"Sample test failed: {e}")
-                    return f"Sample test failed: {e}"
-                except Exception as e:
-                    self.failures.append(f"An error occurred: {e}")
-                    logger.error(f"Sample test failed: {e}")
-                    return f"Sample test failed: {e}"
+        llm_spec = self.llm_specs[spec_index]
+        return await self.test_llm_spec(llm_spec, user_prompt)
 
 
 # Initialize OperatorToolBox with AgentSpecification
@@ -126,104 +107,95 @@ dataset_manager_agent = Agent(
     model="gpt-4",
     deps_type=OperatorToolBox,
     result_type=str,
-    system_prompt="You can validate the toolbox, run operations, and retrieve results or failures.",
+    system_prompt="You can validate the toolbox, run operations, retrieve results or failures, and test LLM specs.",
 )
 
 
 @dataset_manager_agent.tool
 async def validate_toolbox(ctx: RunContext[OperatorToolBox]) -> str:
     is_valid = ctx.deps.validate()
-    if is_valid:
-        return "ToolBox validation successful."
-    else:
-        return "ToolBox validation failed."
+    return (
+        "ToolBox validation successful." if is_valid else "ToolBox validation failed."
+    )
 
 
 @dataset_manager_agent.tool
 async def execute_operation(ctx: RunContext[OperatorToolBox], operation: str) -> str:
-    result = ctx.deps.run_operation(operation)
-    return result
+    return ctx.deps.run_operation(operation)
 
 
 @dataset_manager_agent.tool
 async def retrieve_results(ctx: RunContext[OperatorToolBox]) -> str:
     results = ctx.deps.get_results()
-    if results:
-        formatted_results = "\n".join([f"{op}: {res}" for op, res in results.items()])
-        return f"Operation Results:\n{formatted_results}"
-    else:
-        return "No operations have been executed yet."
+    return (
+        f"Operation Results:\n{results}"
+        if results
+        else "No operations have been executed yet."
+    )
 
 
 @dataset_manager_agent.tool
 async def retrieve_failures(ctx: RunContext[OperatorToolBox]) -> str:
     failures = ctx.deps.get_failures()
-    if failures:
-        formatted_failures = "\n".join(failures)
-        return f"Failures:\n{formatted_failures}"
-    else:
-        return "No failures recorded."
+    return f"Failures:\n{failures}" if failures else "No failures recorded."
 
 
 @dataset_manager_agent.tool
-async def test_agent(
-    ctx: RunContext[OperatorToolBox], description: str, sample_test: dict[str, Any]
+async def list_llm_specs(ctx: RunContext[OperatorToolBox]) -> str:
+    spec_list = "\n".join(
+        f"{i}: {spec.url}" for i, spec in enumerate(ctx.deps.llm_specs)
+    )
+    return f"Available LLM Specs:\n{spec_list}"
+
+
+@dataset_manager_agent.tool
+async def test_llm_with_prompt(
+    ctx: RunContext[OperatorToolBox], spec_index: int, user_prompt: str
 ) -> str:
-    result = await ctx.deps.test(description, sample_test)
-    return result
+    return await ctx.deps.test_with_prompt(spec_index, user_prompt)
 
 
-# Synchronous run example
-def run_dataset_manager_agent_sync():
-    prompts = [
-        "Validate the toolbox.",
-        "Execute operation on 'dataset2'.",
-        "Execute operation on 'dataset4'.",  # This should fail
-        "Retrieve the results.",
-        "Retrieve any failures.",
-        "Test my openAI compatible agent deployed at localhost:3000",
-    ]
-
-    sample_test = {"prompt": "Hello, how are you?", "max_tokens": 5}
-
-    for prompt in prompts:
-        if "Test my" in prompt:
-            result = dataset_manager_agent.run_sync(
-                prompt, deps=toolbox, sample_test=sample_test
-            )
-        else:
-            result = dataset_manager_agent.run_sync(prompt, deps=toolbox)
-        print(f"Prompt: {prompt}")
-        print(f"Response: {result.data}\n")
-
-
-# Asynchronous run example
+# Asynchronous run example with user confirmation
 async def run_dataset_manager_agent_async():
     prompts = [
         "Validate the toolbox.",
-        "Execute operation on 'dataset2'.",
-        "Execute operation on 'dataset4'.",  # This should fail
-        "Retrieve the results.",
-        "Retrieve any failures.",
-        "Test my openAI compatible agent deployed at localhost:3000",
+        "List available LLM specs.",
+        "I want to test an LLM with my prompt: 'Tell me a short story about a robot'. Which spec index should I use?",
     ]
 
-    sample_test = {"prompt": "Hello, how are you?", "max_tokens": 5}
-
     for prompt in prompts:
-        if "Test my" in prompt:
-            result = await dataset_manager_agent.run(
-                prompt, deps=toolbox, sample_test=sample_test
-            )
-        else:
-            result = await dataset_manager_agent.run(prompt, deps=toolbox)
+        result = await dataset_manager_agent.run(prompt, deps=toolbox)
         print(f"Prompt: {prompt}")
         print(f"Response: {result.data}\n")
 
+        # Handle testing request
+        if "test an LLM with my prompt" in prompt:
+            print(
+                "Please select a spec index from the list above and confirm to proceed."
+            )
+            # Simulate user input for demo (in real app, you'd get this from user)
+            user_input = (
+                input("Enter spec index and 'yes' to confirm (e.g., '0 yes'): ")
+                .strip()
+                .split()
+            )
+            if len(user_input) == 2 and user_input[1].lower() == "yes":
+                try:
+                    spec_index = int(user_input[0])
+                    # Extract prompt from the original input
+                    user_prompt = prompt.split("my prompt: ")[1].strip("'")
+                    test_result = await dataset_manager_agent.run(
+                        f"Test LLM at index {spec_index} with prompt: {user_prompt}",
+                        deps=toolbox,
+                        spec_index=spec_index,
+                        user_prompt=user_prompt,
+                    )
+                    print(f"Test Response: {test_result.data}\n")
+                except ValueError:
+                    print("Invalid spec index provided.\n")
+            else:
+                print("Test canceled. Please provide a valid index and confirmation.\n")
+
 
 if __name__ == "__main__":
-    # Run synchronous example
-    run_dataset_manager_agent_sync()
-
-    # Run asynchronous example
     asyncio.run(run_dataset_manager_agent_async())

agentic_security/probe_actor/refusal.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 
-from agentic_security.refusal_classifier import RefusalClassifier
+from agentic_security.refusal_classifier.model import RefusalClassifier
 
 classifier = RefusalClassifier()
 classifier.load_model()

agentic_security/probe_data/__init__.py

@@ -5,6 +5,7 @@ REGISTRY_V0 = [
         "dataset_name": "simonycl/aya-23-8B_advbench_jailbreak",
         "num_prompts": 416,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -16,6 +17,7 @@ REGISTRY_V0 = [
         "dataset_name": "acmc/jailbreaks_dataset_with_perplexity_bigcode_starcoder2-3b_bigcode_starcoder2-7b",
         "num_prompts": 11191,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -27,6 +29,7 @@ REGISTRY_V0 = [
         "dataset_name": "karanxa/dolphin-jailbreak-finetuning-dataset",
         "num_prompts": 42684,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -38,6 +41,7 @@ REGISTRY_V0 = [
         "dataset_name": "karanxa/llama-2-jailbreak-dataset",
         "num_prompts": 40613,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -49,6 +53,7 @@ REGISTRY_V0 = [
         "dataset_name": "karanxa/llama2-uncensored-jailbreak-dataset-finetuning",
         "num_prompts": 42854,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -60,6 +65,7 @@ REGISTRY_V0 = [
         "dataset_name": "liuyanchen1015/Llama-3.2-1B_jailbreak_responses",
         "num_prompts": 9888,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -71,6 +77,7 @@ REGISTRY_V0 = [
         "dataset_name": "liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses",
         "num_prompts": 9888,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -82,6 +89,7 @@ REGISTRY_V0 = [
         "dataset_name": "liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses_with_judgment",
         "num_prompts": 9888,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -93,6 +101,7 @@ REGISTRY_V0 = [
         "dataset_name": "jackhhao/jailbreak-classification",
         "num_prompts": 1044,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -104,6 +113,7 @@ REGISTRY_V0 = [
         "dataset_name": "markush1/LLM-Jailbreak-Classifier",
         "num_prompts": 201193,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -115,6 +125,7 @@ REGISTRY_V0 = [
         "dataset_name": "walledai/JailbreakBench",
         "num_prompts": 200,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -126,6 +137,7 @@ REGISTRY_V0 = [
         "dataset_name": "walledai/JailbreakHub",
         "num_prompts": 15140,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -137,6 +149,7 @@ REGISTRY_V0 = [
         "dataset_name": "Granther/evil-jailbreak",
         "num_prompts": 1200,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -148,6 +161,7 @@ REGISTRY_V0 = [
         "dataset_name": "sevdeawesome/jailbreak_success",
         "num_prompts": 10800,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -159,6 +173,7 @@ REGISTRY_V0 = [
         "dataset_name": "IDA-SERICS/Disaster-tweet-jailbreaking",
         "num_prompts": 3000,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -170,6 +185,7 @@ REGISTRY_V0 = [
         "dataset_name": "GeorgeDaDude/Jailbreak_Complete_DS_labeled",
         "num_prompts": 11383,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -181,6 +197,7 @@ REGISTRY_V0 = [
         "dataset_name": "dayone3nder/jailbreak_prompt_JBB_sft_trainset",
         "num_prompts": 4785,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -192,6 +209,7 @@ REGISTRY_V0 = [
         "dataset_name": "dayone3nder/general_safe_mix_jailbreak_prompt_JBB_trainset",
         "num_prompts": 24679,
         "tokens": None,  # Add actual token count if available
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -206,6 +224,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "AgenticBackend",
         "num_prompts": 2000,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Fine-tuned cloud hosted model",
         "selected": True,
@@ -221,6 +240,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "ShawnMenz/DAN_jailbreak",
         "num_prompts": 666,
         "tokens": 224196,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -232,6 +252,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "deepset/prompt-injections",
         "num_prompts": 203,
         "tokens": 6988,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -243,6 +264,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "rubend18/ChatGPT-Jailbreak-Prompts",
         "num_prompts": 79,
         "tokens": 26971,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -254,6 +276,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "notrichardren/refuse-to-answer-prompts",
         "num_prompts": 522,
         "tokens": 7172,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -265,6 +288,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "Lemhf14/EasyJailbreak_Datasets",
         "num_prompts": 1630,
         "tokens": 19758,
+        "is_active": False,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -276,6 +300,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "markush1/LLM-Jailbreak-Classifier",
         "num_prompts": 1119,
         "tokens": 19758,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -287,6 +312,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "JailbreakV-28K/JailBreakV-28k",
         "num_prompts": 28300,
         "tokens": 1975800,
+        "is_active": False,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -298,6 +324,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "ShawnMenz/jailbreak_sft_rm_ds",
         "num_prompts": 371000,
         "tokens": 1975800,
+        "is_active": False,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
         "selected": False,
@@ -309,6 +336,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "Steganography",
         "num_prompts": 10,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Local mutation dataset",
         "selected": False,
@@ -320,6 +348,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "GPT fuzzer",
         "num_prompts": 10,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Local mutation dataset",
         "selected": False,
@@ -331,6 +360,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "jailbreak_llms/2023_05_07",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Github",
         "selected": False,
@@ -342,6 +372,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "jailbreak_llms/2023_12_25.csv",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Github",
         "selected": False,
@@ -353,6 +384,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "Malwaregen",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Local dataset",
         "selected": False,
@@ -364,6 +396,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "Hallucination",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Local dataset",
         "selected": False,
@@ -375,6 +408,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "DataLeak",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Local dataset",
         "selected": False,
@@ -386,6 +420,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "llm-adaptive-attacks",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Github: tml-epfl/llm-adaptive-attacks#0.0.1",
         "selected": False,
@@ -397,6 +432,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "Garak",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Github: https://github.com/leondz/garak#v0.9.0.1",
         "selected": False,
@@ -412,6 +448,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "Reinforcement Learning Optimization",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Cloud hosted model",
         "selected": False,
@@ -427,6 +464,7 @@ REGISTRY = REGISTRY_V0 + [
         "dataset_name": "InspectAI",
         "num_prompts": 0,
         "tokens": 0,
+        "is_active": True,
         "approx_cost": 0.0,
         "source": "Github: https://github.com/UKGovernmentBEIS/inspect_ai",
         "selected": False,
@@ -439,6 +477,7 @@ REGISTRY = REGISTRY_V0 + [
         "num_prompts": len(load_local_csv().prompts),
         "tokens": load_local_csv().tokens,
         "approx_cost": 0.0,
+        "is_active": True,
         "source": f"Local file dataset: {load_local_csv().metadata['src']}",
         "selected": len(load_local_csv().prompts),
         "url": "",

agentic_security/probe_data/audio_generator.py

@@ -1,4 +1,5 @@
 import base64
+import logging
 import os
 import platform
 import subprocess
@@ -7,6 +8,16 @@ import uuid
 import httpx
 from cache_to_disk import cache_to_disk
 
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class AudioGenerationError(Exception):
+    """Custom exception for errors during audio generation."""
+
+    pass
+
 
 def encode(content: bytes) -> str:
     encoded_content = base64.b64encode(content).decode("utf-8")
@@ -41,12 +52,25 @@ def generate_audio_mac_wav(prompt: str) -> bytes:
         # Read the WAV file into memory
         with open(temp_wav_path, "rb") as f:
             audio_bytes = f.read()
+
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Subprocess error: {e}")
+        raise AudioGenerationError("Failed to generate or convert audio.") from e
+    except FileNotFoundError as e:
+        logger.error(f"File not found: {e}")
+        raise AudioGenerationError("Required file not found.") from e
+    except Exception as e:
+        logger.exception("Unexpected error occurred.")
+        raise AudioGenerationError(
+            "An unexpected error occurred during audio generation."
+        ) from e
     finally:
-        # Clean up the temporary files
-        if os.path.exists(temp_aiff_path):
-            os.remove(temp_aiff_path)
-        if os.path.exists(temp_wav_path):
-            os.remove(temp_wav_path)
+        for path in (temp_aiff_path, temp_wav_path):
+            try:
+                if os.path.exists(path):
+                    os.remove(path)
+            except Exception as e:
+                logger.warning(f"Failed to delete temporary file {path}: {e}")
 
     # Return the audio bytes
     return audio_bytes

agentic_security/probe_data/data.py

@@ -1,14 +1,15 @@
 import io
 import os
 import random
-from functools import lru_cache
+from collections.abc import Callable, Iterator
+from functools import partial
 
 import httpx
 import pandas as pd
 from cache_to_disk import cache_to_disk
 from datasets import load_dataset
-from loguru import logger
 
+from agentic_security.logutils import logger
 from agentic_security.probe_data import stenography_fn
 from agentic_security.probe_data.models import ProbeDataset
 from agentic_security.probe_data.modules import (
@@ -19,281 +20,200 @@ from agentic_security.probe_data.modules import (
     rl_model,
 )
 
+# Type aliases for clarity
+FilterFn = Callable[[pd.Series], bool]
+ColumnMappings = dict[str, str]
+DatasetLoader = Callable[[], ProbeDataset]
 
-@cache_to_disk()
-def load_dataset_general(
-    dataset_name,
-    dataset_split="train",
-    column_mappings=None,
-    filter_fn=None,
-    custom_url=None,
-    additional_metadata=None,
-):
-    """
-    Generalized function to load datasets with flexible configurations.
 
-    :param dataset_name: Name of the dataset or URL for custom CSVs
-    :param dataset_split: Split to load from the dataset (e.g., "train")
-    :param column_mappings: Dictionary mapping dataset columns to expected keys, e.g., {'prompt': 'query'}
-    :param filter_fn: A filtering function that takes a row and returns True/False
-    :param custom_url: URL for custom CSV datasets
-    :param additional_metadata: Additional metadata to include in the ProbeDataset
-    :return: A ProbeDataset object with the processed data
-    """
-    if custom_url:
-        logger.info(f"Loading custom CSV dataset from {custom_url}")
-        r = httpx.get(custom_url)
-        content = r.content
-        df = pd.read_csv(io.StringIO(content.decode("utf-8")))
-    else:
-        logger.info(f"Loading dataset {dataset_name} from Hugging Face datasets")
-        dataset = load_dataset(dataset_name)
-        df = pd.DataFrame(dataset[dataset_split])
+# Core data loading utilities
+def fetch_csv_content(url: str) -> str:
+    """Fetch CSV content from a URL."""
+    response = httpx.get(url)
+    return response.content.decode("utf-8")
 
-    # Apply column mappings if provided
-    if column_mappings:
-        df.rename(columns=column_mappings, inplace=True)
 
-    # Filter rows if filter_fn is provided
+def load_df_from_source(source: str, is_url: bool = False) -> pd.DataFrame:
+    """Load DataFrame from either URL or Hugging Face dataset."""
+    if is_url:
+        logger.info(f"Loading custom CSV from {source}")
+        return pd.read_csv(io.StringIO(fetch_csv_content(source)))
+    logger.info(f"Loading dataset {source} from Hugging Face")
+    return pd.DataFrame(load_dataset(source)["train"])
+
+
+def transform_df(
+    df: pd.DataFrame,
+    mappings: ColumnMappings | None = None,
+    filter_fn: FilterFn | None = None,
+) -> pd.DataFrame:
+    """Apply transformations to DataFrame."""
+    result = df.copy()
+    if mappings:
+        result.rename(columns=mappings, inplace=True)
     if filter_fn:
-        df = df[df.apply(filter_fn, axis=1)]
-
-    # Extract prompts
-    prompts = df[column_mappings.get("prompt", "prompt")].tolist()
-
-    return ProbeDataset(
-        dataset_name=dataset_name,
-        metadata=additional_metadata or {},
-        prompts=prompts,
-        tokens=count_words_in_list(prompts),
-        approx_cost=0.0,
-    )
+        result = result[result.apply(filter_fn, axis=1)]
+    return result
 
 
-def count_words_in_list(str_list):
-    """Calculate the total number of words in a given list of strings.
-
-    :param str_list: List of strings
-    :return: Total number of words across all strings in the list
-    """
-    total_words = sum(len(str(s).split()) for s in str_list)
-    return total_words
-
-
-@cache_to_disk()
-def load_dataset_v1():
-    return load_dataset_general(
-        dataset_name="ShawnMenz/DAN_jailbreak",
-        column_mappings={"prompt": "prompt", "jailbreak": "jailbreak"},
-        filter_fn=lambda row: row["jailbreak"],
-    )
-
-
-@cache_to_disk()
-def load_dataset_v2():
-    return load_dataset_general(
-        dataset_name="deepset/prompt-injections",
-        column_mappings={"prompt": "text", "jailbreak": "label"},
-        filter_fn=lambda row: row["label"],
-    )
-
-
-@cache_to_disk()
-def load_dataset_v4():
-    return load_dataset_general(
-        dataset_name="notrichardren/refuse-to-answer-prompts",
-        column_mappings={"prompt": "claim", "jailbreak": "label"},
-        filter_fn=lambda row: row["label"],
-    )
-
-
-@cache_to_disk()
-def load_dataset_v3():
-    return load_dataset_general(
-        dataset_name="rubend18/ChatGPT-Jailbreak-Prompts",
-        column_mappings={"prompt": "Prompt"},
-        filter_fn=lambda row: row["label"],
-    )
-
-
-@cache_to_disk()
-def load_dataset_v6():
-    dataset = load_dataset("markush1/LLM-Jailbreak-Classifier")
-    bad_prompts = [
-        p
-        for p, c in zip(dataset["train"]["prompt"], dataset["train"]["classification"])
-        if c == "jailbreak"
-    ]
-    return ProbeDataset(
-        dataset_name="markush1/LLM-Jailbreak-Classifier",
-        metadata={},
-        prompts=bad_prompts,
-        tokens=count_words_in_list(bad_prompts),
-        approx_cost=0.0,
-    )
-
-
-@cache_to_disk()
-def load_dataset_v7():
-    splits = {
-        "mini_JailBreakV_28K": "JailBreakV_28K/mini_JailBreakV_28K.csv",
-        "JailBreakV_28K": "JailBreakV_28K/JailBreakV_28K.csv",
-    }
-    df = pd.read_csv(
-        "hf://datasets/JailbreakV-28K/JailBreakV-28k/" + splits["JailBreakV_28K"]
-    )
-    bad_prompts = df["jailbreak_query"].tolist()
-    print(df.shape)
-    return ProbeDataset(
-        dataset_name="JailbreakV-28K/JailBreakV-28k",
-        metadata={},
-        prompts=bad_prompts,
-        tokens=count_words_in_list(bad_prompts),
-        approx_cost=0.0,
-    )
-
-
-@cache_to_disk()
-def load_dataset_v8():
-    df = pd.read_csv(
-        "hf://datasets/ShawnMenz/jailbreak_sft_rm_ds/jailbreak_sft_rm_ds.csv",
-        names=["jailbreak", "prompt"],
-    )
-    filtered = df[df["jailbreak"] == "jailbreak"]["prompt"].tolist()
-    return ProbeDataset(
-        dataset_name="JailbreakV-28K/JailBreakV-28k",
-        metadata={},
-        prompts=filtered,
-        tokens=count_words_in_list(filtered),
-        approx_cost=0.0,
-    )
-
-
-@cache_to_disk()
-def load_dataset_v5():
-    ds = []
-    for c in [
-        "AdvBench",
-        "ForbiddenQuestion",
-        "MJP",
-        "MaliciousInstruct",
-        "QuestionList",
-    ]:
-        dataset = load_dataset("Lemhf14/EasyJailbreak_Datasets", c)
-        bad_prompts = dataset["train"]["query"]
-        ds.extend(bad_prompts)
-
-    return ProbeDataset(
-        dataset_name="Lemhf14/EasyJailbreak_Datasets",
-        metadata={},
-        prompts=ds,
-        tokens=count_words_in_list(ds),
-        approx_cost=0.0,
-    )
-
-
-@cache_to_disk()
-def load_generic_csv(url, name, column="prompt", predicator=None):
-    r = httpx.get(url)
-    content = r.content
-    df = pd.read_csv(io.StringIO(content.decode("utf-8")))
-    logger.info(f"Loaded {len(df)} prompts from {url}")
-    filtered_prompts = df[df.apply(predicator, axis=1)][column].tolist()
+def create_probe_dataset(
+    name: str, prompts: list[str], metadata: dict = None
+) -> ProbeDataset:
+    """Create a ProbeDataset from prompts."""
+    metadata = metadata or {}
     return ProbeDataset(
         dataset_name=name,
-        metadata={},
-        prompts=filtered_prompts,
-        tokens=count_words_in_list(filtered_prompts),
+        metadata=metadata,
+        prompts=prompts,
+        tokens=sum(len(str(s).split()) for s in prompts),
         approx_cost=0.0,
     )
 
 
-def prepare_prompts(dataset_names, budget, tools_inbox=None, options=[]):
-    # ## Datasets used and cleaned:
-    # markush1/LLM-Jailbreak-Classifier
-    # 1. Open-Orca/OpenOrca
-    # 2. ShawnMenz/DAN_jailbreak
-    # 3. EddyLuo/JailBreakV_28K
-    # 4. https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/jailbreak_prompts.csv
-
-    dataset_map = {
-        "ShawnMenz/DAN_jailbreak": load_dataset_v1,
-        "deepset/prompt-injections": load_dataset_v2,
-        "notrichardren/refuse-to-answer-prompts": load_dataset_v4,
-        "rubend18/ChatGPT-Jailbreak-Prompts": load_dataset_v3,
-        "Lemhf14/EasyJailbreak_Datasets": load_dataset_v5,
-        "markush1/LLM-Jailbreak-Classifier": load_dataset_v6,
-        "JailbreakV-28K/JailBreakV-28k": load_dataset_v7,
-        "ShawnMenz/jailbreak_sft_rm_ds": load_dataset_v8,
-        "verazuo/jailbreak_llms/2023_05_07": lambda: load_generic_csv(
-            url="https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/prompts/jailbreak_prompts_2023_05_07.csv",
-            name="verazuo/jailbreak_llms/2023_05_07",
-            column="prompt",
-            predicator=lambda x: bool(x["jailbreak"]),
-        ),
-        "verazuo/jailbreak_llms/2023_12_25.csv": lambda: load_generic_csv(
-            url="https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/prompts/jailbreak_prompts_2023_12_25.csv.csv",
-            name="verazuo/jailbreak_llms/2023_12_25.csv",
-            column="prompt",
-            predicator=lambda x: bool(x["jailbreak"]),
-        ),
-        "Custom CSV": load_local_csv,
-    }
-    dataset_map.update(dataset_map_generics)
-
-    group = []
-    for dataset_name in dataset_names:
-        if dataset_name in dataset_map:
-            logger.info(f"Loading {dataset_name}")
-            try:
-                group.append(dataset_map[dataset_name]())
-            except Exception as e:
-                logger.error(f"Error loading {dataset_name}: {e}")
-
-    dynamic_datasets = {
-        "AgenticBackend": lambda opts: dataset_from_iterator(
-            "AgenticBackend",
-            fine_tuned.Module(group, tools_inbox=tools_inbox, opts=opts).apply(),
-            lazy=True,
-        ),
-        "Steganography": lambda opts: Stenography(group),
-        "llm-adaptive-attacks": lambda opts: dataset_from_iterator(
-            "llm-adaptive-attacks",
-            adaptive_attacks.Module(group, tools_inbox=tools_inbox, opts=opts).apply(),
-        ),
-        "Garak": lambda opts: dataset_from_iterator(
-            "Garak",
-            garak_tool.Module(group, tools_inbox=tools_inbox, opts=opts).apply(),
-            lazy=True,
-        ),
-        "Reinforcement Learning Optimization": lambda opts: dataset_from_iterator(
-            "Reinforcement Learning Optimization",
-            rl_model.Module(group, tools_inbox=tools_inbox, opts=opts).apply(),
-            lazy=True,
-        ),
-        "InspectAI": lambda opts: dataset_from_iterator(
-            "InspectAI",
-            inspect_ai_tool.Module(group, tools_inbox=tools_inbox).apply(),
-            lazy=True,
-        ),
-        "GPT fuzzer": lambda opts: [],
-    }
-
-    dynamic_groups = []
-    options = options or [{} for _ in dataset_names]
-    for dataset_name, opts in zip(dataset_names, options):
-        if dataset_name in dynamic_datasets:
-            logger.info(f"Loading {dataset_name}")
-
-            ds = dynamic_datasets[dataset_name](opts)
-
-            for g in ds:
-                dynamic_groups.append(g)
-    return group + dynamic_groups
+# Generalized dataset loader
+@cache_to_disk()
+def load_dataset_generic(
+    name: str,
+    mappings: ColumnMappings | None = None,
+    filter_fn: FilterFn | None = None,
+    url: str | None = None,
+    metadata: dict | None = None,
+) -> ProbeDataset:
+    """Load and process a dataset with flexible configuration."""
+    df = load_df_from_source(url or name, is_url=bool(url))
+    transformed_df = transform_df(df, mappings, filter_fn)
+    prompt_col = mappings.get("prompt", "prompt") if mappings else "prompt"
+    prompts = transformed_df[prompt_col].tolist()
+    return create_probe_dataset(name, prompts, metadata)
 
 
-class Stenography:
-    fn_library = {
+# Dataset-specific configurations
+DATASET_CONFIGS = {
+    "ShawnMenz/DAN_jailbreak": {
+        "mappings": {"prompt": "prompt"},
+        "filter_fn": lambda row: row["jailbreak"],
+    },
+    "deepset/prompt-injections": {
+        "mappings": {"prompt": "text"},
+        "filter_fn": lambda row: row["label"],
+    },
+    "notrichardren/refuse-to-answer-prompts": {
+        "mappings": {"prompt": "claim"},
+        "filter_fn": lambda row: row["label"],
+    },
+    "rubend18/ChatGPT-Jailbreak-Prompts": {
+        "mappings": {"prompt": "Prompt"},
+        "filter_fn": lambda row: row["label"],
+    },
+    "markush1/LLM-Jailbreak-Classifier": {
+        "mappings": {"prompt": "prompt"},
+        "filter_fn": lambda row: row["classification"] == "jailbreak",
+    },
+    "ShawnMenz/jailbreak_sft_rm_ds": {
+        "url": "hf://datasets/ShawnMenz/jailbreak_sft_rm_ds/jailbreak_sft_rm_ds.csv",
+        "mappings": {"prompt": "prompt"},
+        "filter_fn": lambda row: row["jailbreak"] == "jailbreak",
+    },
+    "verazuo/jailbreak_llms/2023_05_07": {
+        "url": "https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/prompts/jailbreak_prompts_2023_05_07.csv",
+        "mappings": {"prompt": "prompt"},
+        "filter_fn": lambda row: bool(row["jailbreak"]),
+    },
+    "verazuo/jailbreak_llms/2023_12_25": {
+        "url": "https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/prompts/jailbreak_prompts_2023_12_25.csv",
+        "mappings": {"prompt": "prompt"},
+        "filter_fn": lambda row: bool(row["jailbreak"]),
+    },
+}
+
+# Additional generic dataset configurations
+DATASET_CONFIGS_GENERICS = {
+    "simonycl/aya-23-8B_advbench_jailbreak": {"mappings": {"prompt": "prompt"}},
+    "acmc/jailbreaks_dataset_with_perplexity_bigcode_starcoder2-3b_bigcode_starcoder2-7b": {},
+    "karanxa/dolphin-jailbreak-finetuning-dataset": {"mappings": {"prompt": "text"}},
+    "karanxa/llama-2-jailbreak-dataset": {"mappings": {"prompt": "text"}},
+    "karanxa/llama2-uncensored-jailbreak-dataset-finetuning": {
+        "mappings": {"prompt": "text"}
+    },
+    "liuyanchen1015/Llama-3.2-1B_jailbreak_responses": {
+        "mappings": {"prompt": "jailbreak_prompt_text"}
+    },
+    "liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses": {
+        "mappings": {"prompt": "jailbreak_prompt_text"}
+    },
+    "liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses_with_judgment": {
+        "mappings": {"prompt": "jailbreak_prompt_text"}
+    },
+    "jackhhao/jailbreak-classification": {"mappings": {"prompt": "prompt"}},
+    "walledai/JailbreakBench": {"mappings": {"prompt": "prompt"}},
+    "walledai/JailbreakHub": {"mappings": {"prompt": "prompt"}},
+    "Granther/evil-jailbreak": {"mappings": {"prompt": "text"}},
+    "sevdeawesome/jailbreak_success": {"mappings": {"prompt": "jailbreak_prompt_text"}},
+    "IDA-SERICS/Disaster-tweet-jailbreaking": {"mappings": {"prompt": "prompt_attack"}},
+    "GeorgeDaDude/Jailbreak_Complete_DS_labeled": {"mappings": {"prompt": "question"}},
+    "dayone3nder/jailbreak_prompt_JBB_sft_trainset": {"mappings": {"prompt": "prompt"}},
+    "dayone3nder/general_safe_mix_jailbreak_prompt_JBB_trainset": {
+        "mappings": {"prompt": "prompt"}
+    },
+}
+
+
+# Dataset factory
+def create_dataset_loader(name: str, config: dict) -> DatasetLoader:
+    """Create a dataset loader from configuration."""
+    return partial(
+        load_dataset_generic,
+        name=name,
+        mappings=config.get("mappings"),
+        filter_fn=config.get("filter_fn"),
+        url=config.get("url"),
+    )
+
+
+# Specialized loaders
+@cache_to_disk()
+def load_multi_dataset(name: str, sub_datasets: list[str]) -> ProbeDataset:
+    """Load and combine multiple sub-datasets."""
+    prompts = []
+    for sub in sub_datasets:
+        dataset = load_dataset(name, sub)
+        prompts.extend(dataset["train"]["query"])
+    return create_probe_dataset(f"{name}_combined", prompts)
+
+
+@cache_to_disk()
+def load_jailbreak_v28k() -> ProbeDataset:
+    """Load JailBreakV-28K dataset."""
+    df = pd.read_csv("hf://datasets/JailbreakV-28K/JailBreakV-28k/JailBreakV_28K.csv")
+    prompts = df["jailbreak_query"].tolist()
+    return create_probe_dataset("JailbreakV-28K/JailBreakV-28k", prompts)
+
+
+@cache_to_disk()
+def load_local_csv() -> ProbeDataset:
+    """Load prompts from local CSV files."""
+    csv_files = [f for f in os.listdir(".") if f.endswith(".csv")]
+    logger.info(f"Found {len(csv_files)} CSV files: {csv_files}")
+
+    prompts = []
+    for file in csv_files:
+        try:
+            df = pd.read_csv(file)
+            if "prompt" in df.columns:
+                prompts.extend(df["prompt"].tolist())
+            else:
+                logger.warning(f"File {file} lacks 'prompt' column")
+        except Exception as e:
+            logger.error(f"Error reading {file}: {e}")
+
+    return create_probe_dataset("Local CSV", prompts, {"src": str(csv_files)})
+
+
+# Stenography transformer
+class StenographyTransformer:
+    """Apply stenography transformations to datasets."""
+
+    TRANSFORMATIONS = {
         "rot5": stenography_fn.rot5,
         "rot13": stenography_fn.rot13,
         "base64": stenography_fn.base64_encode,
@@ -306,71 +226,32 @@ class Stenography:
         "zigzag_obfuscation": stenography_fn.zigzag_obfuscation,
     }
 
-    def __init__(self, prompt_groups: [ProbeDataset]):
-        self.prompt_groups = prompt_groups
+    def __init__(self, datasets: list[ProbeDataset]):
+        self.datasets = datasets
 
-    def __iter__(self):
-        return self.apply()
+    def transform(self) -> Iterator[ProbeDataset]:
+        """Generate transformed datasets."""
+        for dataset in self.datasets:
+            size = len(dataset.prompts)
+            sample_size = int(size * 0.25)
 
-    def apply(self):
-        for prompt_group in self.prompt_groups:
-            size = len(prompt_group.prompts)
-            for name, fn in self.fn_library.items():
-                logger.info(f"Applying {name} to {prompt_group.dataset_name}")
-                selected_prompts = random.sample(prompt_group.prompts, int(size * 0.25))
-                prompts = [
-                    f"Follow the instructions in {name}: {fn(p)}"
-                    for p in selected_prompts
+            for name, fn in self.TRANSFORMATIONS.items():
+                logger.info(f"Applying {name} to {dataset.dataset_name}")
+                sampled = random.sample(dataset.prompts, sample_size)
+                transformed = [
+                    f"Follow the instructions in {name}: {fn(p)}" for p in sampled
                 ]
-                yield ProbeDataset(
-                    dataset_name=f"stenography.{name}({prompt_group.dataset_name})",
-                    metadata={},
-                    prompts=prompts,
-                    tokens=count_words_in_list(prompts),
-                    approx_cost=0.0,
+                yield create_probe_dataset(
+                    f"stenography.{name}({dataset.dataset_name})", transformed
                 )
 
 
-@lru_cache
-def load_local_csv() -> ProbeDataset:
-    csv_files = [f for f in os.listdir(".") if f.endswith(".csv")]
-    logger.info(f"Found {len(csv_files)} CSV files")
-    logger.info(f"CSV files: {csv_files}")
-    prompt_list = []
-
-    for file in csv_files:
-        try:
-            df = pd.read_csv(file)
-        except Exception as e:
-            logger.error(f"Error reading {file}: {e}")
-            continue
-        # Check if 'prompt' column exists
-        if "prompt" in df.columns:
-            prompt_list.extend(df["prompt"].tolist())
-        else:
-            logger.warning(f"File {file} does not contain a 'prompt' column")
-    return ProbeDataset(
-        dataset_name="Local CSV",
-        metadata={"src": str(csv_files)},
-        prompts=prompt_list,
-        tokens=count_words_in_list(prompt_list),
-        approx_cost=0.0,
-    )
-
-
-def dataset_from_iterator(name: str, iterator, lazy=False) -> list:
-    """Convert an iterator into a list of prompts and create a ProbeDataset
-    object.
-
-    Args:
-        name (str): The name of the dataset.
-        iterator (iterator): An iterator that generates prompts.
-
-    Returns:
-        list: A list containing a single ProbeDataset object.
-    """
+def dataset_from_iterator(
+    name: str, iterator, lazy: bool = False
+) -> list[ProbeDataset]:
+    """Convert an iterator into a list of ProbeDataset objects."""
     prompts = list(iterator) if not lazy else iterator
-    tokens = count_words_in_list(prompts) if not lazy else 0
+    tokens = sum(len(str(s).split()) for s in prompts) if not lazy else 0
     dataset = ProbeDataset(
         dataset_name=name,
         metadata={},
@@ -382,75 +263,85 @@ def dataset_from_iterator(name: str, iterator, lazy=False) -> list:
     return [dataset]
 
 
-# TODO: refactor this abstraction
+# Main dataset preparation
+def prepare_prompts(
+    dataset_names: list[str],
+    budget: float,
+    tools_inbox=None,
+    options: list[dict] = None,
+) -> list[ProbeDataset]:
+    """Prepare datasets based on names and options."""
+    # Base dataset loaders
+    dataset_loaders = {
+        **{k: create_dataset_loader(k, v) for k, v in DATASET_CONFIGS.items()},
+        **{k: create_dataset_loader(k, v) for k, v in DATASET_CONFIGS_GENERICS.items()},
+        "Lemhf14/EasyJailbreak_Datasets": partial(
+            load_multi_dataset,
+            name="Lemhf14/EasyJailbreak_Datasets",
+            sub_datasets=[
+                "AdvBench",
+                "ForbiddenQuestion",
+                "MJP",
+                "MaliciousInstruct",
+                "QuestionList",
+            ],
+        ),
+        "JailbreakV-28K/JailBreakV-28k": load_jailbreak_v28k,
+        "Local CSV": load_local_csv,
+    }
 
-dataset_map_generics = {
-    "simonycl/aya-23-8B_advbench_jailbreak": lambda: load_dataset_general(
-        dataset_name="simonycl/aya-23-8B_advbench_jailbreak",
-        column_mappings={"prompt": "prompt"},
-    ),
-    "acmc/jailbreaks_dataset_with_perplexity_bigcode_starcoder2-3b_bigcode_starcoder2-7b": lambda: load_dataset_general(
-        dataset_name="acmc/jailbreaks_dataset_with_perplexity_bigcode_starcoder2-3b_bigcode_starcoder2-7b"
-    ),
-    "karanxa/dolphin-jailbreak-finetuning-dataset": lambda: load_dataset_general(
-        dataset_name="karanxa/dolphin-jailbreak-finetuning-dataset",
-        column_mappings={"prompt": "text"},
-    ),
-    "karanxa/llama-2-jailbreak-dataset": lambda: load_dataset_general(
-        dataset_name="karanxa/llama-2-jailbreak-dataset",
-        column_mappings={"prompt": "text"},
-    ),
-    "karanxa/llama2-uncensored-jailbreak-dataset-finetuning": lambda: load_dataset_general(
-        dataset_name="karanxa/llama2-uncensored-jailbreak-dataset-finetuning",
-        column_mappings={"prompt": "text"},
-    ),
-    "liuyanchen1015/Llama-3.2-1B_jailbreak_responses": lambda: load_dataset_general(
-        dataset_name="liuyanchen1015/Llama-3.2-1B_jailbreak_responses",
-        column_mappings={"prompt": "jailbreak_prompt_text"},
-    ),
-    "liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses": lambda: load_dataset_general(
-        dataset_name="liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses",
-        column_mappings={"prompt": "jailbreak_prompt_text"},
-    ),
-    "liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses_with_judgment": lambda: load_dataset_general(
-        dataset_name="liuyanchen1015/Llama-3.2-1B-Instruct_jailbreak_responses_with_judgment",
-        column_mappings={"prompt": "jailbreak_prompt_text"},
-    ),
-    "jackhhao/jailbreak-classification": lambda: load_dataset_general(
-        dataset_name="jackhhao/jailbreak-classification",
-        column_mappings={"prompt": "prompt"},
-    ),
-    "markush1/LLM-Jailbreak-Classifier": lambda: load_dataset_general(
-        dataset_name="markush1/LLM-Jailbreak-Classifier",
-        column_mappings={"prompt": "prompt"},
-    ),
-    "walledai/JailbreakBench": lambda: load_dataset_general(
-        dataset_name="walledai/JailbreakBench", column_mappings={"prompt": "prompt"}
-    ),
-    "walledai/JailbreakHub": lambda: load_dataset_general(
-        dataset_name="walledai/JailbreakHub", column_mappings={"prompt": "prompt"}
-    ),
-    "Granther/evil-jailbreak": lambda: load_dataset_general(
-        dataset_name="Granther/evil-jailbreak", column_mappings={"prompt": "text"}
-    ),
-    "sevdeawesome/jailbreak_success": lambda: load_dataset_general(
-        dataset_name="sevdeawesome/jailbreak_success",
-        column_mappings={"prompt": "jailbreak_prompt_text"},
-    ),
-    "IDA-SERICS/Disaster-tweet-jailbreaking": lambda: load_dataset_general(
-        dataset_name="IDA-SERICS/Disaster-tweet-jailbreaking",
-        column_mappings={"prompt": "prompt_attack"},
-    ),
-    "GeorgeDaDude/Jailbreak_Complete_DS_labeled": lambda: load_dataset_general(
-        dataset_name="GeorgeDaDude/Jailbreak_Complete_DS_labeled",
-        column_mappings={"prompt": "question"},
-    ),
-    "dayone3nder/jailbreak_prompt_JBB_sft_trainset": lambda: load_dataset_general(
-        dataset_name="dayone3nder/jailbreak_prompt_JBB_sft_trainset",
-        column_mappings={"prompt": "prompt"},
-    ),
-    "dayone3nder/general_safe_mix_jailbreak_prompt_JBB_trainset": lambda: load_dataset_general(
-        dataset_name="dayone3nder/general_safe_mix_jailbreak_prompt_JBB_trainset",
-        column_mappings={"prompt": "prompt"},
-    ),
-}
+    # Dynamic dataset loaders
+    dynamic_loaders = {
+        "AgenticBackend": lambda opts: dataset_from_iterator(
+            "AgenticBackend",
+            fine_tuned.Module([], tools_inbox=tools_inbox, opts=opts).apply(),
+            lazy=True,
+        ),
+        "Steganography": lambda opts: list(StenographyTransformer([]).transform()),
+        "llm-adaptive-attacks": lambda opts: dataset_from_iterator(
+            "llm-adaptive-attacks",
+            adaptive_attacks.Module([], tools_inbox=tools_inbox, opts=opts).apply(),
+        ),
+        "Garak": lambda opts: dataset_from_iterator(
+            "Garak",
+            garak_tool.Module([], tools_inbox=tools_inbox, opts=opts).apply(),
+            lazy=True,
+        ),
+        "Reinforcement Learning Optimization": lambda opts: dataset_from_iterator(
+            "Reinforcement Learning Optimization",
+            rl_model.Module([], tools_inbox=tools_inbox, opts=opts).apply(),
+            lazy=True,
+        ),
+        "InspectAI": lambda opts: dataset_from_iterator(
+            "InspectAI",
+            inspect_ai_tool.Module([], tools_inbox=tools_inbox).apply(),
+            lazy=True,
+        ),
+        "GPT fuzzer": lambda opts: [],
+    }
+
+    options = options or [{} for _ in dataset_names]
+    datasets = []
+
+    # Load base datasets
+    for name, opts in zip(dataset_names, options):
+        if name in dataset_loaders:
+            logger.info(f"Loading base dataset {name}")
+            try:
+                datasets.append(dataset_loaders[name]())
+            except Exception as e:
+                logger.error(f"Error loading {name}: {e}")
+
+    # Load dynamic datasets and apply transformations
+    for name, opts in zip(dataset_names, options):
+        if name in dynamic_loaders:
+            logger.info(f"Loading dynamic dataset {name}")
+            try:
+                dynamic_result = dynamic_loaders[name](opts)
+                datasets.extend(dynamic_result)
+            except Exception as e:
+                logger.error(f"Error loading dynamic {name}: {e}")
+        elif name == "Steganography":
+            datasets.extend(list(StenographyTransformer(datasets).transform()))
+
+    return datasets

agentic_security/probe_data/modules/adaptive_attacks.py

@@ -3,7 +3,8 @@ import io
 
 import httpx
 import pandas as pd
-from loguru import logger
+
+from agentic_security.logutils import logger
 
 url = "https://raw.githubusercontent.com/tml-epfl/llm-adaptive-attacks/main/harmful_behaviors/harmful_behaviors_pair.csv"
 

agentic_security/probe_data/modules/fine_tuned.py

@@ -3,7 +3,8 @@ import os
 import uuid as U
 
 import httpx
-from loguru import logger
+
+from agentic_security.logutils import logger
 
 AUTH_TOKEN: str = os.getenv("AS_TOKEN", "gh0-5f4a8ed2-37c6-4bd7-a0cf-7070eae8115b")
 
@@ -65,7 +66,7 @@ class Module:
                 return {}
 
     async def fetch_prompts(self) -> list[str]:
-        api_url = "https://edge.metaheuristic.co/infer"
+        api_url = "https://mcp.metaheuristic.co/infer"
         headers = {
             "Authorization": f"Bearer {AUTH_TOKEN}",
             "Content-Type": "application/json",

agentic_security/probe_data/modules/garak_tool.py

@@ -4,7 +4,7 @@ import json
 import os
 import subprocess
 
-from loguru import logger
+from agentic_security.logutils import logger
 
 # TODO: add probes modules
 

agentic_security/probe_data/modules/inspect_ai_tool.py

@@ -2,7 +2,7 @@ import asyncio
 import importlib.util
 import os
 
-from loguru import logger
+from agentic_security.logutils import logger
 
 inspect_ai_task = (
     __file__.replace("inspect_ai_tool.py", "inspect_ai_task.py")

agentic_security/probe_data/modules/rl_model.py

@@ -8,7 +8,8 @@ from typing import Deque
 
 import numpy as np
 import requests
-from loguru import logger
+
+from agentic_security.logutils import logger
 
 AUTH_TOKEN: str = os.getenv("AS_TOKEN", "gh0-5f4a8ed2-37c6-4bd7-a0cf-7070eae8115b")
 
@@ -215,7 +216,7 @@ class Module:
         self.run_id = U.uuid4().hex
         self.batch_size = self.opts.get("batch_size", 500)
         self.rl_model = CloudRLPromptSelector(
-            prompt_groups, "https://edge.metaheuristic.co", run_id=self.run_id
+            prompt_groups, "https://mcp.metaheuristic.co", run_id=self.run_id
         )
 
     async def apply(self):

agentic_security/probe_data/modules/test_rl_model.py

@@ -36,7 +36,7 @@ def mock_requests() -> Mock:
 def mock_rl_selector() -> Mock:
     return CloudRLPromptSelector(
         dataset_prompts,
-        api_url="https://edge.metaheuristic.co",
+        api_url="https://mcp.metaheuristic.co",
     )
 
 
@@ -94,7 +94,7 @@ class TestCloudRLPromptSelector:
     def test_select_next_prompt_success_service(self, dataset_prompts):
         selector = CloudRLPromptSelector(
             dataset_prompts,
-            api_url="https://edge.metaheuristic.co",
+            api_url="https://mcp.metaheuristic.co",
         )
         next_prompt = selector.select_next_prompt(
             "How does RL work?", passed_guard=True

agentic_security/report_chart.py

@@ -7,9 +7,65 @@ import pandas as pd
 from matplotlib.cm import ScalarMappable
 from matplotlib.colors import LinearSegmentedColormap, Normalize
 
+from agentic_security.logutils import logger
 
-def plot_security_report(table):
+from .primitives import Table
+
+
+def plot_security_report(table: Table) -> io.BytesIO:
+    """
+    Generates a polar plot representing the security report based on the given data.
+
+    Args:
+        table (Table): The input data table containing security metrics.
+
+    Returns:
+        io.BytesIO: A buffer containing the generated plot image in PNG format.
+                    Returns an empty buffer in case of an error.
+    """
+    try:
+        return _plot_security_report(table=table)
+    except (TypeError, ValueError, OverflowError, IndexError, Exception) as e:
+        logger.error(f"Error in generating the security report: {e}")
+    return io.BytesIO()
+
+
+def generate_identifiers(data: pd.DataFrame) -> list[str]:
+    """
+    Generates unique identifiers for the given dataset.
+
+    Args:
+        data (pd.DataFrame): A pandas DataFrame containing security-related data.
+
+    Returns:
+        list[str]: A list of generated identifiers. Returns a list with an empty string in case of an error.
+    """
+    try:
+        _generate_identifiers(data=data)
+    except (TypeError, ValueError, Exception) as e:
+        logger.error(f"Error in generate_identifiers: {e}")
+    return [""]
+
+
+def _plot_security_report(table: Table) -> io.BytesIO:
+    """
+    Generates a polar plot-based security report visualizing the failure rates for different modules.
+
+    This function processes the input data, sorts it by failure rate, and generates a polar plot
+    where each bar represents the failure rate for a specific module. The plot includes identifiers,
+    color-coding based on token count, failure rate values on the bars, and a table listing the modules
+    and their corresponding failure rates.
+
+    Args:
+        table (Table): A table-like structure (e.g., pandas DataFrame) containing security report data
+                        with columns for failure rate, tokens, and modules.
+
+    Returns:
+        io.BytesIO: A buffer containing the generated plot image in PNG format.
+    """
     # Data preprocessing
+    logger.info("Data preprocessing started.")
+
     data = pd.DataFrame(table)
 
     # Sort by failure rate and reset index
@@ -20,10 +76,10 @@ def plot_security_report(table):
     fig, ax = plt.subplots(figsize=(12, 10), subplot_kw={"projection": "polar"})
     fig.set_facecolor("#f0f0f0")
     ax.set_facecolor("#f0f0f0")
+    logger.info("Plot setup complete.")
 
     # Styling parameters
     colors = ["#6C5B7B", "#C06C84", "#F67280", "#F8B195"][::-1]  # Pastel palette
-    # colors = ["#440154", "#3b528b", "#21908c", "#5dc863"]  # Viridis-inspired palette
     cmap = LinearSegmentedColormap.from_list("custom", colors, N=256)
     norm = Normalize(vmin=data["tokens"].min(), vmax=data["tokens"].max())
 
@@ -74,7 +130,10 @@ def plot_security_report(table):
 
     # Title and caption
     fig.suptitle(
-        "Security Report for Different Modules", fontsize=16, fontweight="bold", y=1.02
+        "Security Report for Different Modules",
+        fontsize=16,
+        fontweight="bold",
+        y=1.02,
     )
     caption = "Report generated by https://github.com/msoedov/agentic_security"
     fig.text(
@@ -112,17 +171,12 @@ def plot_security_report(table):
             data["identifier"], data["failureRate"], data["module"]
         )
     ]
-    table = ax.table(
-        cellText=table_data,
-        loc="right",
-        cellLoc="left",
-    )
+    table = ax.table(cellText=table_data, loc="right", cellLoc="left")
     table.auto_set_font_size(False)
     table.set_fontsize(8)
 
     # Adjust table style
     table.scale(1, 0.7)
-
     for (row, col), cell in table.get_celld().items():
         cell.set_edgecolor("none")
         cell.set_facecolor("#f0f0f0" if row % 2 == 0 else "#e0e0e0")
@@ -132,17 +186,33 @@ def plot_security_report(table):
             cell.set_text_props(fontweight="bold")
 
     # Adjust layout and save
-
     plt.tight_layout()
     buf = io.BytesIO()
     plt.savefig(buf, format="png", dpi=300, bbox_inches="tight")
     plt.close(fig)
     buf.seek(0)
+    logger.info("Report successfully generated and saved to buffer.")
     return buf
 
 
-def generate_identifiers(data):
+def _generate_identifiers(data: pd.DataFrame) -> list[str]:
+    """
+    Generates a list of unique identifiers for each row in the given DataFrame.
+
+    The identifiers are based on the English alphabet, with each identifier consisting
+    of a letter followed by a number. The letter represents the "group" of identifiers
+    (using a letter from A to Z) and the number is a counter within that group. For example:
+    - A1, A2, ..., A26, B1, B2, ..., Z1, Z2, ...
+
+    Args:
+        data (pd.DataFrame): The input DataFrame containing data for which identifiers
+                             are to be generated.
+
+    Returns:
+        list[str]: A list of unique identifiers as strings, one for each row in the DataFrame.
+    """
     data_length = len(data)
+
     alphabet = string.ascii_uppercase
     num_letters = len(alphabet)
 

agentic_security/routes/probe.py

@@ -3,7 +3,7 @@ import random
 from fastapi import APIRouter, File, Header, HTTPException, UploadFile
 from fastapi.responses import JSONResponse
 
-from ..models.schemas import FileProbeResponse, Probe
+from ..primitives import FileProbeResponse, Probe
 from ..probe_actor.refusal import REFUSAL_MARKS
 from ..probe_data import REGISTRY
 
@@ -77,3 +77,11 @@ async def data_config():
 async def health_check():
     """Health check endpoint."""
     return JSONResponse(content={"status": "ok"})
+
+
+@router.post("/v1/self-probe-t5")
+def self_probe_t5(probe: Probe):
+    import languagemodels as lm  # noqa
+
+    message = lm.do(probe.prompt)
+    return make_mock_response(message)

agentic_security/routes/proxy.py

@@ -2,10 +2,11 @@ import random
 from asyncio import Event
 
 from fastapi import APIRouter
-from loguru import logger
+
+from agentic_security.logutils import logger
 
 from ..core.app import get_current_run, get_tools_inbox
-from ..models.schemas import CompletionRequest, Settings
+from ..primitives import CompletionRequest, Settings
 from ..probe_actor.refusal import REFUSAL_MARKS
 
 router = APIRouter()

agentic_security/routes/report.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from fastapi import APIRouter, Response
 from fastapi.responses import FileResponse, StreamingResponse
 
-from ..models.schemas import Table
+from ..primitives import Table
 from ..report_chart import plot_security_report
 
 router = APIRouter()

agentic_security/routes/scan.py

@@ -1,4 +1,6 @@
+from collections.abc import Generator
 from datetime import datetime
+from typing import Any
 
 from fastapi import (
     APIRouter,
@@ -11,10 +13,12 @@ from fastapi import (
 )
 from fastapi.responses import StreamingResponse
 
+from agentic_security.logutils import logger
+
 from ..core.app import get_stop_event, get_tools_inbox, set_current_run
 from ..dependencies import InMemorySecrets, get_in_memory_secrets
 from ..http_spec import LLMSpec
-from ..models.schemas import LLMInfo, Scan
+from ..primitives import LLMInfo, Scan
 from ..probe_actor import fuzzer
 
 router = APIRouter()
@@ -23,9 +27,14 @@ router = APIRouter()
 @router.post("/verify")
 async def verify(
     info: LLMInfo, secrets: InMemorySecrets = Depends(get_in_memory_secrets)
-):
+) -> dict[str, int | str | float]:
     spec = LLMSpec.from_string(info.spec)
-    r = await spec.verify()
+    try:
+        r = await spec.verify()
+    except Exception as e:
+        logger.exception(e)
+        raise HTTPException(status_code=400, detail=str(e))
+
     if r.status_code >= 400:
         raise HTTPException(status_code=r.status_code, detail=r.text)
     return dict(
@@ -36,7 +45,7 @@ async def verify(
     )
 
 
-def streaming_response_generator(scan_parameters: Scan):
+def streaming_response_generator(scan_parameters: Scan) -> Generator[str, Any, None]:
     request_factory = LLMSpec.from_string(scan_parameters.llmSpec)
     set_current_run(request_factory)
 
@@ -57,7 +66,7 @@ async def scan(
     scan_parameters: Scan,
     background_tasks: BackgroundTasks,
     secrets: InMemorySecrets = Depends(get_in_memory_secrets),
-):
+) -> StreamingResponse:
     scan_parameters.with_secrets(secrets)
     return StreamingResponse(
         streaming_response_generator(scan_parameters), media_type="application/json"
@@ -65,7 +74,7 @@ async def scan(
 
 
 @router.post("/stop")
-async def stop_scan():
+async def stop_scan() -> dict[str, str]:
     get_stop_event().set()
     return {"status": "Scan stopped"}
 
@@ -79,7 +88,7 @@ async def scan_csv(
     maxBudget: int = Query(10_000),
     enableMultiStepAttack: bool = Query(False),
     secrets: InMemorySecrets = Depends(get_in_memory_secrets),
-):
+) -> StreamingResponse:
     # TODO: content dataset to fuzzer
     content = await file.read()  # noqa
     llm_spec = await llmSpec.read()

agentic_security/routes/static.py

@@ -7,7 +7,7 @@ from fastapi.templating import Jinja2Templates
 from jinja2 import Environment, FileSystemLoader
 from starlette.responses import Response
 
-from ..models.schemas import Settings
+from ..primitives import Settings
 
 router = APIRouter()
 STATIC_DIR = Path(__file__).parent.parent / "static"

agentic_security/routes/telemetry.py

@@ -1,8 +1,9 @@
 import sentry_sdk
-from loguru import logger
 from sentry_sdk.integrations.logging import ignore_logger
 
-from ..models.schemas import Settings
+from agentic_security.logutils import logger
+
+from ..primitives import Settings
 
 
 def setup(app):
@@ -16,6 +17,7 @@ def setup(app):
         # Set traces_sample_rate to 1.0 to capture 100%
         # of transactions for tracing.
         traces_sample_rate=1.0,
+        ignore_errors=[KeyboardInterrupt],
         _experiments={
             # Set continuous_profiling_auto_start to True
             # to automatically start the profiler on when

agentic_security/static/base.js

@@ -1,4 +1,3 @@
-
 let SELF_URL = window.location.href;
 if (SELF_URL.endsWith('/')) {
   SELF_URL = SELF_URL.slice(0, -1);
@@ -102,7 +101,7 @@ Content-Type: application/json
 ]
 `,
   `POST ${SELF_URL}/v1/self-probe-file
-Authorization: Bearer $GROQ_API_KEY
+Authorization: Bearer $API_KEY
 Content-Type: multipart/form-data
 
 {
@@ -171,6 +170,21 @@ Content-Type: application/json
 {
   "audio_url": "<<AUDIO_FILE_URL>>"
 }
+`,
+
+  `POST https://api.openrouter.ai/v1/chat/completions
+Authorization: Bearer $OPENROUTER_API_KEY
+Content-Type: application/json
+
+{
+  "model": "openrouter-latest",
+  "prompt": "<<PROMPT>>",
+  "temperature": 0.7,
+  "max_tokens": 150,
+  "top_p": 0.9,
+  "frequency_penalty": 0,
+  "presence_penalty": 0
+}
 `,
 
 ]
@@ -191,6 +205,8 @@ let LLM_CONFIGS = [
   { name: 'Cohere', prompts: 40000, logo: '/icons/cohere.png' },
   { name: 'Azure OpenAI', prompts: 40000, logo: '/icons/azureai.png' },
   { name: 'assemblyai', prompts: 40000, logo: fallbackIcon },
+  { name: 'OpenRouter.ai', prompts: 40000, logo: '/icons/openrouter.png' },
+
 ];
 function has_image(spec) {
   return spec.includes('<<BASE64_IMAGE>>');
@@ -226,5 +242,6 @@ function _getFailureRateScore(failureRate) {
   else if (strengthRate >= 80) return 'B';
   else if (strengthRate >= 70) return 'C';
   else if (strengthRate >= 60) return 'D';
+  else if (strengthRate >= 1) return '?';
   else return 'E'; // For strengthRate less than 60
 }

agentic_security/static/index.html

@@ -63,6 +63,32 @@
           </div>
         </section>
       </main>
+      <div class="fixed top-6 right-6 z-50 space-y-3">
+        <transition-group name="toast">
+            <div
+                v-for="toast in toasts"
+                :key="toast.id"
+                class="flex items-center p-3 rounded-xl shadow-xl text-white max-w-md animate-toast-in border border-opacity-30"
+                :class="{
+                    'bg-success-toast border-accent-green': toast.type === 'success',
+                    'bg-error-toast border-accent-red': toast.type === 'error',
+                    'bg-info-toast border-accent-orange': toast.type === 'info'
+                }"
+            >
+                <span class="flex-1 font-medium tracking-wide text-sm">{{ toast.message }}</span>
+                <button
+                    @click="removeToast(toast.id)"
+                    class="ml-3 focus:outline-none hover:opacity-80 transition-opacity"
+                >
+                    <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
+                    </svg>
+                </button>
+            </div>
+        </transition-group>
+    </div>
+
+
       <main class="max-w-6xl mx-auto space-y-8">
         <section class="bg-dark-card rounded-lg p-6 shadow-lg" v-show="false">
           <h2 class="text-2xl font-bold mb-4">Select a Config</h2>
@@ -383,27 +409,26 @@
                 class="text-gray-400 hover:underline">Deselect All</button>
             </div>
 
-            <div class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
-              <div
-                v-for="(package, index) in dataConfig"
-                :key="index"
-                @click="addPackage(index)"
-                class="border rounded-lg p-3 cursor-pointer transition-all hover:shadow-md overflow-hidden"
-                :class="{
-                'border-dark-accent-green bg-dark-accent-green bg-opacity-20': package.selected,
-                'border-gray-600': !package.selected
-              }">
-                <div class="font-medium mb-1 truncate">{{ package.dataset_name
-                  }}</div>
-                <div class="text-sm text-gray-400 truncate">
-                  {{ package.source || 'Local dataset' }}
-                </div>
-                <div class="mt-2 text-sm font-semibold">
-                  {{ package.dynamic ? 'Dynamic dataset' :
-                  `${package.num_prompts.toLocaleString()} prompts` }}
-                </div>
-              </div>
+          <div class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
+          <div
+            v-for="(package, index) in dataConfig"
+            :key="index"
+            @click="package.is_active !== false && addPackage(index)"
+            class="border rounded-lg p-3 cursor-pointer transition-all hover:shadow-md overflow-hidden"
+            :class="{
+              'border-dark-accent-green bg-dark-accent-green bg-opacity-20': package.selected,
+              'border-gray-600': !package.selected,
+              'opacity-30 pointer-events-none cursor-not-allowed': package.is_active === false
+            }">
+            <div class="font-medium mb-1 truncate">{{ package.dataset_name }}</div>
+            <div class="text-sm text-gray-400 truncate">
+              {{ package.source || 'Local dataset' }}
             </div>
+            <div class="mt-2 text-sm font-semibold">
+              {{ package.dynamic ? 'Dynamic dataset' : `${package.num_prompts.toLocaleString()} prompts` }}
+            </div>
+          </div>
+        </div>
           </div>
         </section>
 

agentic_security/static/main.js

@@ -25,6 +25,8 @@ var app = new Vue({
         showModules: false,
         showLogs: false,
         showConsentModal: true,
+        toasts: [], // Array to store toast notifications
+        toastTimeout: 3000, // Duration in milliseconds (3 seconds)
         statusDotClass: 'bg-gray-500', // Default status dot class
         statusText: 'Verified', // Default status text
         statusClass: 'bg-green-500 text-dark-bg', // Default status class
@@ -93,6 +95,19 @@ var app = new Vue({
 
     },
     methods: {
+        showToast(message, type = 'success') {
+            const id = Date.now(); // Unique ID for each toast
+            this.toasts.push({ id, message, type });
+
+            // Automatically remove toast after timeout
+            setTimeout(() => {
+                this.removeToast(id);
+            }, this.toastTimeout);
+        },
+
+        removeToast(id) {
+            this.toasts = this.toasts.filter(toast => toast.id !== id);
+        },
         focusTextarea() {
             this.isFocused = true;
             self = this.$refs;
@@ -210,10 +225,12 @@ var app = new Vue({
             if (!response.ok) {
                 this.updateStatusDot(false);
                 this.errorMsg = 'Integration verification failed:' + JSON.stringify(r);
+                this.showToast('Integration verification failed', 'error');
             } else {
                 this.errorMsg = '';
                 this.updateStatusDot(true);
                 this.okMsg = 'Integration verified';
+                this.showToast('Integration verified successfully', 'success');
                 this.integrationVerified = true;
                 // console.log('Integration verified', this.integrationVerified);
                 // this.$forceUpdate();
@@ -350,6 +367,10 @@ var app = new Vue({
 
             // If all are selected, deselect all. Otherwise, select all.
             this.dataConfig.forEach(package => {
+                if (!package.is_active) {
+                    package.selected = false;
+                    return
+                }
                 package.selected = !allSelected;
             });
 

agentic_security/static/partials/head.html

@@ -19,7 +19,7 @@
               technopollas: ['Technopollas', 'sans-serif'],
             },
             colors: {
-                t1: {
+                dark: {
                 bg: '#0D0D0D', // Jet Black
                 card: '#1A1A1A', // Dark Carbon Fiber
                 text: '#FFFFFF',
@@ -30,7 +30,7 @@
                   yellow: '#2E4053', // Jet Black
                 },
               },
-              dark: {
+              dark1: {
                 bg: '#121212',
                 card: '#1E1E1E',
                 text: '#FFFFFF',
@@ -86,6 +86,7 @@
         }
       }
     </script>
+
     <style>
 .scrollbar-hide::-webkit-scrollbar {
   display: none;
@@ -96,4 +97,55 @@
 }
 
     </style>
+<style>
+/* Toast-specific colors */
+.bg-success-toast {
+    background: #1C3F74
+}
+
+.bg-error-toast {
+    background: #85144B
+}
+
+.bg-info-toast {
+    background: #FFC300
+}
+
+.border-accent-green, .border-accent-red, .border-accent-orange {
+    border-color: rgba(255, 255, 255, 0.1); /* Subtle white border for depth */
+}
+
+/* Animation */
+.animate-toast-in {
+    animation: tSlideIn 0.4s cubic-bezier(0.25, 0.1, 0.25, 1);
+}
+
+@keyframes tSlideIn {
+    from {
+        transform: translateX(120%) scale(0.95);
+        opacity: 0;
+    }
+    to {
+        transform: translateX(0) scale(1);
+        opacity: 1;
+    }
+}
+
+.toast-enter-active,
+.toast-leave-active {
+    transition: all 0.4s cubic-bezier(0.25, 0.1, 0.25, 1);
+}
+
+.toast-enter-from,
+.toast-leave-to {
+    transform: translateX(120%) scale(0.95);
+    opacity: 0;
+}
+
+/* Hover effect */
+[toast-type]:hover {
+    transform: translateY(-2px);
+    box-shadow: 0 6px 20px rgba(0, 0, 0, 0.6);
+}
+</style>
   </head>

docs/http_spec.md

@@ -33,7 +33,7 @@ The `LLMSpec` class is the core of the HTTP specification. It provides the follo
 ### Methods
 
 - **`from_string(http_spec: str) -> LLMSpec`**: Parses an HTTP specification string into an `LLMSpec` object.
-- **`validate(prompt: str, encoded_image: str, encoded_audio: str, files: dict) -> None`**: Validates the request parameters based on the specified modality.
+- **`validate(prompt: str, encoded_image: str, encoded_audio: str, files: dict) -> null`**: Validates the request parameters based on the specified modality.
 - **`probe(prompt: str, encoded_image: str = "", encoded_audio: str = "", files: dict = {}) -> httpx.Response`**: Sends an HTTP request using the specified parameters.
 - **`verify() -> httpx.Response`**: Verifies the HTTP specification by sending a test request.
 
@@ -52,12 +52,11 @@ Authorization: Bearer sk-xxxxxxxxx
 Content-Type: application/json
 
 {
-    "model": "gpt-3.5-turbo",
-    "messages": [{"role": "user", "content": "<<PROMPT>>"}],
-    "temperature": 0.7
+  "model": "gpt-3.5-turbo",
+  "messages": [{"role": "user", "content": "<<PROMPT>>"}],
+  "temperature": 0.7
 }
 """
-
 spec = LLMSpec.from_string(http_spec)
 response = await spec.probe("What is the capital of France?")
 ```
@@ -71,12 +70,11 @@ Authorization: Bearer sk-xxxxxxxxx
 Content-Type: application/json
 
 {
-    "model": "gpt-4-vision-preview",
-    "messages": [{"role": "user", "content": "What is in this image? <<BASE64_IMAGE>>"}],
-    "temperature": 0.7
+  "model": "gpt-4-vision-preview",
+  "messages": [{"role": "user", "content": "What is in this image? <<BASE64_IMAGE>>"}],
+  "temperature": 0.7
 }
 """
-
 spec = LLMSpec.from_string(http_spec)
 encoded_image = encode_image_base64_by_url("https://example.com/image.jpg")
 response = await spec.probe("What is in this image?", encoded_image=encoded_image)
@@ -91,12 +89,11 @@ Authorization: Bearer sk-xxxxxxxxx
 Content-Type: application/json
 
 {
-    "model": "whisper-large-v3",
-    "messages": [{"role": "user", "content": "Transcribe this audio: <<BASE64_AUDIO>>"}],
-    "temperature": 0.7
+  "model": "whisper-large-v3",
+  "messages": [{"role": "user", "content": "Transcribe this audio: <<BASE64_AUDIO>>"}],
+  "temperature": 0.7
 }
 """
-
 spec = LLMSpec.from_string(http_spec)
 encoded_audio = encode_audio_base64_by_url("https://example.com/audio.mp3")
 response = await spec.probe("Transcribe this audio:", encoded_audio=encoded_audio)
@@ -111,12 +108,11 @@ Authorization: Bearer sk-xxxxxxxxx
 Content-Type: multipart/form-data
 
 {
-    "model": "gpt-3.5-turbo",
-    "messages": [{"role": "user", "content": "Process this file: <<FILE>>"}],
-    "temperature": 0.7
+  "model": "gpt-3.5-turbo",
+  "messages": [{"role": "user", "content": "Process this file: <<FILE>>"}],
+  "temperature": 0.7
 }
 """
-
 spec = LLMSpec.from_string(http_spec)
 files = {"file": ("document.txt", open("document.txt", "rb"))}
 response = await spec.probe("Process this file:", files=files)

docs/index.md

@@ -21,4 +21,4 @@ Note: Please be aware that Agentic Security is designed as a safety scanner tool
 
 ## UI 🧙
 
-<img width="100%" alt="booking-screen" src="https://res.cloudinary.com/dq0w2rtm9/image/upload/v1736433557/z0bsyzhsqlgcr3w4ovwp.gif">
+<img width="100%" alt="booking-screen" src="https://res.cloudinary.com/dq0w2rtm9/image/upload/v1741192668/final_aa9jhb.gif">

docs/probe_data.md

@@ -54,20 +54,15 @@ The `probe_data` module is a core component of the Agentic Security project, res
 
 - **Classes:**
   - `PromptSelectionInterface`: Abstract base class for prompt selection strategies.
-
     - Methods:
       - `select_next_prompt(current_prompt: str, passed_guard: bool) -> str`: Selects next prompt
       - `select_next_prompts(current_prompt: str, passed_guard: bool) -> list[str]`: Selects multiple prompts
-      - `update_rewards(previous_prompt: str, current_prompt: str, reward: float, passed_guard: bool) -> None`: Updates rewards
-
+      - `update_rewards(previous_prompt: str, current_prompt: str, reward: float, passed_guard: bool) -> null`: Updates rewards
   - `RandomPromptSelector`: Basic random selection with history tracking.
-
     - Parameters:
       - `prompts: list[str]`: List of available prompts
       - `history_size: int = 3`: Size of history to prevent cycles
-
   - `CloudRLPromptSelector`: Cloud-based RL implementation with fallback.
-
     - Parameters:
       - `prompts: list[str]`: List of available prompts
       - `api_url: str`: URL of RL service
@@ -75,9 +70,7 @@ The `probe_data` module is a core component of the Agentic Security project, res
       - `history_size: int = 300`: Size of history
       - `timeout: int = 5`: Request timeout
       - `run_id: str = ""`: Unique run identifier
-
   - `QLearningPromptSelector`: Local Q-learning implementation.
-
     - Parameters:
       - `prompts: list[str]`: List of available prompts
       - `learning_rate: float = 0.1`: Learning rate
@@ -86,13 +79,11 @@ The `probe_data` module is a core component of the Agentic Security project, res
       - `exploration_decay: float = 0.995`: Exploration decay rate
       - `min_exploration: float = 0.01`: Minimum exploration rate
       - `history_size: int = 300`: Size of history
-
-  - `Module`: Main class that uses CloudRLPromptSelector.
-
-    - Parameters:
-      - `prompt_groups: list[str]`: Groups of prompts
-      - `tools_inbox: asyncio.Queue`: Queue for tool communication
-      - `opts: dict = {}`: Configuration options
+- **Module**: Main class that uses CloudRLPromptSelector.
+  - Parameters:
+    - `prompt_groups: list[str]`: Groups of prompts
+    - `tools_inbox: asyncio.Queue`: Queue for tool communication
+    - `opts: dict = {}`: Configuration options
 
 ## Usage Examples
 
@@ -119,10 +110,9 @@ from agentic_security.probe_data.modules.rl_model import QLearningPromptSelector
 
 prompts = ["What is AI?", "Explain machine learning"]
 selector = QLearningPromptSelector(prompts)
-
 current_prompt = "What is AI?"
-next_prompt = selector.select_next_prompt(current_prompt, passed_guard=True)
-selector.update_rewards(current_prompt, next_prompt, reward=1.0, passed_guard=True)
+next_prompt = selector.select_next_prompt(current_prompt, passed_guard=true)
+selector.update_rewards(current_prompt, next_prompt, reward=1.0, passed_guard=true)
 ```
 
 ## Conclusion

docs/stylesheets/extra.css

@@ -1,14 +1,16 @@
 :root {
-  --md-primary-fg-color: #e92063;
-  --md-primary-fg-color--light: #e92063;
-  --md-primary-fg-color--dark: #e92063;
+  --md-primary-fg-color: #2E4053;
+  /* Primary color changed to pinkish */
+  --md-primary-fg-color--light: #E0A3B6;
+  --md-primary-fg-color--dark: #1C3F74;
+  /* Dark variant changed to blue */
 }
 
-
-/* Revert hue value to that of pre mkdocs-material v9.4.0 */
+/* Updated slate color scheme with new background */
 [data-md-color-scheme="slate"] {
   --md-hue: 230;
-  --md-default-bg-color: hsla(230, 15%, 21%, 1);
+  --md-default-bg-color: #1A1A1A;
+  /* Background changed to dark gray */
 }
 
 .hide {
@@ -24,12 +26,15 @@ img.index-header {
   max-width: 500px;
 }
 
+/* Updated custom colors */
 .pydantic-pink {
-  color: #FF007F;
+  color: #E0A3B6;
+  /* Updated to match new theme */
 }
 
 .team-blue {
-  color: #0072CE;
+  color: #1C3F74;
+  /* Updated to match new theme */
 }
 
 .secure-green {
@@ -67,7 +72,6 @@ img.index-header {
   text-align: center;
 }
 
-
 /* Hide the entire footer */
 .md-footer {
   display: none;

mkdocs.yml

@@ -89,7 +89,7 @@ theme:
         name: Switch to light mode
   icon:
     repo: fontawesome/brands/github
-  favicon: "https://res.cloudinary.com/dq0w2rtm9/image/upload/v1737555066/r17hrkre246doczwmvbv.png"
+  favicon: https://res.cloudinary.com/dq0w2rtm9/image/upload/v1741195421/favicon_kuz6xr.png
 
 extra:
   generator: false

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agentic_security"
-version = "0.5.1"
+version = "0.7.0"
 description = "Agentic LLM vulnerability scanner"
 authors = ["Alexander Miasoiedov <msoedov@gmail.com>"]
 maintainers = ["Alexander Miasoiedov <msoedov@gmail.com>"]
@@ -49,7 +49,12 @@ tomli = "^2.2.1"
 rich = "13.9.4"
 gTTS = "^2.5.4"
 sentry_sdk = "^2.22.0"
+orjson = "^3.10"
+pyfiglet = "^1.0.2"
+termcolor = "^2.4.0"
+
 # garak = { version = "*", optional = true }
+pytest-xdist = "3.6.1"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -82,5 +87,7 @@ build-backend = "poetry.core.masonry.api"
 
 
 [tool.pytest.ini_options]
+addopts = "--durations=5 -m 'not slow' -n 3"
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
+markers = "slow: marks tests as slow"

tests/conftest.py

@@ -0,0 +1,8 @@
+import os
+
+import pytest
+
+
+def pytest_runtest_setup(item):
+    if "slow" in item.keywords and not os.getenv("RUN_SLOW_TESTS"):
+        pytest.skip("Skipping slow test")

tests/probe_actor/test_fuzzer.py

@@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
 import httpx
 import pytest
 
-from agentic_security.models.schemas import Scan
+from agentic_security.primitives import Scan
 from agentic_security.probe_actor.fuzzer import (
     generate_prompts,
     perform_many_shot_scan,

tests/refusal_classifier/test_model.py

@@ -3,7 +3,7 @@ from unittest.mock import patch
 import pandas as pd
 import pytest
 
-from .model import RefusalClassifier
+from agentic_security.refusal_classifier.model import RefusalClassifier
 
 
 @pytest.fixture

tests/routes/test_health.py

@@ -1,6 +1,6 @@
 from fastapi.testclient import TestClient
 
-from ..app import app
+from agentic_security.app import app
 
 
 def test_health_check():

tests/routes/test_probe.py

@@ -5,10 +5,10 @@ import httpx
 import pytest
 from fastapi.testclient import TestClient
 
-from ..app import app
-from ..models.schemas import Probe
-from ..probe_actor.refusal import REFUSAL_MARKS
-from ..probe_data import REGISTRY
+from agentic_security.app import app
+from agentic_security.primitives import Probe
+from agentic_security.probe_actor.refusal import REFUSAL_MARKS
+from agentic_security.probe_data import REGISTRY
 
 client = TestClient(app)
 

tests/routes/test_report.py

@@ -4,7 +4,7 @@ from unittest.mock import patch
 import pytest
 from fastapi.testclient import TestClient
 
-from .report import router
+from agentic_security.routes.report import router
 
 client = TestClient(router)
 

tests/routes/test_static.py

@@ -4,8 +4,8 @@ import pytest
 from fastapi import HTTPException
 from fastapi.testclient import TestClient
 
-from ..models.schemas import Settings
-from .static import get_static_file, router
+from agentic_security.primitives import Settings
+from agentic_security.routes.static import get_static_file, router
 
 client = TestClient(router)
 

tests/test_lib.py

@@ -8,7 +8,7 @@ import time
 import pytest
 
 import agentic_security.test_spec_assets as test_spec_assets
-from agentic_security.lib import AgenticSecurity
+from agentic_security.lib import SecurityScanner
 
 
 def has_module(module_name):
@@ -50,25 +50,25 @@ def make_test_registry():
     ]
 
 
-class TestAS:
+class TestLibraryLevel:
     # Handles an empty dataset list.
     def test_class(self, test_server):
         llmSpec = test_spec_assets.SAMPLE_SPEC
         maxBudget = 1000000
         max_th = 0.3
         datasets = make_test_registry()
-        result = AgenticSecurity.scan(llmSpec, maxBudget, datasets, max_th)
+        result = SecurityScanner.scan(llmSpec, maxBudget, datasets, max_th)
         assert isinstance(result, dict)
         print(result)
         assert len(result) in [0, 1]
 
-    # TODO: slow test
-    def _test_class_msj(self, test_server):
+    @pytest.mark.slow
+    def test_class_msj(self, test_server):
         llmSpec = test_spec_assets.SAMPLE_SPEC
         maxBudget = 1000
         max_th = 0.3
         datasets = make_test_registry()
-        result = AgenticSecurity.scan(
+        result = SecurityScanner.scan(
             llmSpec, maxBudget, datasets, max_th, enableMultiStepAttack=True
         )
         assert isinstance(result, dict)
@@ -93,11 +93,12 @@ class TestAS:
                 "opts": {"port": 9094},
             },
         ]
-        result = AgenticSecurity.scan(llmSpec, maxBudget, datasets, max_th)
+        result = SecurityScanner.scan(llmSpec, maxBudget, datasets, max_th)
         assert isinstance(result, dict)
         print(result)
         assert len(result) in [0, 1]
 
+    @pytest.mark.slow
     def test_backend(self, test_server):
         llmSpec = test_spec_assets.SAMPLE_SPEC
         maxBudget = 1000000
@@ -119,7 +120,7 @@ class TestAS:
                 "modality": "text",
             },
         ]
-        result = AgenticSecurity.scan(llmSpec, maxBudget, datasets, max_th)
+        result = SecurityScanner.scan(llmSpec, maxBudget, datasets, max_th)
         assert isinstance(result, dict)
         print(result)
         assert len(result) in [0, 1]
@@ -147,7 +148,7 @@ class TestAS:
                 "modality": "text",
             },
         ]
-        result = AgenticSecurity.scan(llmSpec, maxBudget, datasets, max_th)
+        result = SecurityScanner.scan(llmSpec, maxBudget, datasets, max_th)
         assert isinstance(result, dict)
         print(result)
         assert len(result) in [0, 1]
@@ -156,18 +157,18 @@ class TestAS:
 class TestEntrypointCI:
     def test_generate_default_cfg_to_tmp_path(self):
         """
-        Test that the `generate_default_cfg` method generates a valid default config file in a temporary path.
+        Test that the `generate_default_settings` method generates a valid default config file in a temporary path.
         """
         # Create a temporary directory
         with tempfile.TemporaryDirectory() as tmpdir:
             temp_path = os.path.join(tmpdir, "custom_agesec.toml")
 
             # Override default_path to the temporary path
-            AgenticSecurity.default_path = temp_path
+            SecurityScanner.default_path = temp_path
 
             # Generate the default configuration
-            security = AgenticSecurity()
-            security.generate_default_cfg()
+            security = SecurityScanner()
+            security.generate_default_settings()
 
             # Check that the config file was created at the temporary path
             assert os.path.exists(temp_path), f"{temp_path} file should be generated."
@@ -188,17 +189,17 @@ class TestEntrypointCI:
             temp_path = os.path.join(tmpdir, "custom_agesec.toml")
 
             # Override default_path to the temporary path
-            AgenticSecurity.default_path = temp_path
+            SecurityScanner.default_path = temp_path
 
             # Generate the default configuration
-            security = AgenticSecurity()
-            security.generate_default_cfg()
+            security = SecurityScanner()
+            security.generate_default_settings()
 
             # Load the generated configuration
-            AgenticSecurity.load_config(temp_path)
+            SecurityScanner.load_config(temp_path)
 
             # Validate loaded configuration
-            config = AgenticSecurity.config
+            config = SecurityScanner.config
             assert (
                 config["general"]["maxBudget"] == 1000000
             ), "maxBudget should be 1000000"

tests/test_registry.py

@@ -0,0 +1,26 @@
+import pytest
+from datasets import load_dataset
+
+from agentic_security.probe_data import REGISTRY
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("dataset", REGISTRY)
+def test_registry_accessibility(dataset):
+    source = dataset.get("source", "")
+    if "hugging" not in source.lower():
+        return pytest.skip("skipped dataset")
+    if not dataset.get("is_active"):
+        return pytest.skip("skipped dataset")
+
+    dataset_name = dataset.get("dataset_name")
+    if not dataset_name:
+        pytest.fail(f"No dataset_name found in {dataset}")
+
+    # Load only metadata (no data download)
+    try:
+        ds = load_dataset(dataset_name, split=None)
+        # Check if metadata is accessible without loading full data
+        assert ds is not None, f"Failed to load metadata for {dataset_name}"
+    except Exception as e:
+        pytest.fail(f"Error loading metadata for {dataset_name}: {str(e)}")