feat(bump 0.5.0):

feat(operator): add agent testing functionality with endpoint

.dockerignore

@@ -1,2 +1,45 @@
-.git/
+# Byte-compiled / optimized / DLL files
 __pycache__/
+*.py[cod]
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+
+# Virtual environments
+
+.venv/
+env/
+ENV/
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# PyInstaller
+*.spec
+
+# macOS specific files
+.DS_Store
+
+# Windows specific files
+Thumbs.db
+desktop.ini
+
+# Tools and editors
+.idea/
+.vscode/
+cmder/
+
+# Output directories
+Output/
+te/

.gitattributes

@@ -0,0 +1,3 @@
+*.js linguist-detectable=false
+*.html linguist-detectable=false
+*.py linguist-detectable=true

.github/workflows/pre-commit.yml

@@ -0,0 +1,21 @@
+name: Pre-Commit Checks
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Install pre-commit
+        run: pip install pre-commit
+      - name: Run pre-commit
+        run: pre-commit run --all-files

.gitignore

@@ -11,3 +11,9 @@ sandbox.py
 site/
 agesec.toml
 .clinerules
+garak_rest.json
+2025.*.json
+inv/
+scripts/
+docx/
+agentic_security.toml

.pre-commit-config.yaml

@@ -43,17 +43,24 @@ repos:
     -   id: check-shebang-scripts-are-executable
     -   id: check-added-large-files
         args: ['--maxkb=100']
+    -   id: trailing-whitespace
+        types: [python]
+    -   id: end-of-file-fixer
+        types: [file]
+        files: \.(py|js|vue)$
 
-  - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
-    hooks:
-      - id: mdformat
-        name: mdformat
-        entry: mdformat .
-        language_version: python3.11
+
+  # - repo: https://github.com/executablebooks/mdformat
+  #   rev: 0.7.22
+  #   hooks:
+  #     - id: mdformat
+  #       name: mdformat
+  #       entry: mdformat .
+  #       language_version: python3.11
+  #       files: "docs/.*\\.md$"
 
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: pycln
 
@@ -75,8 +82,8 @@ repos:
     rev: v2.2.6
     hooks:
     -   id: codespell
-        exclude: '^(third_party/)|(poetry.lock)'
+        exclude: '^(third_party/)|(poetry.lock)|(ui/package-lock.json)|(agentic_security/static/.*)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
-        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie
+        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,vEw
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US

Dockerfile

@@ -21,6 +21,10 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Runtime stage
 FROM python:3.11-slim
 
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
 WORKDIR /app
 
 # Copy only the necessary files from the builder stage

Readme.md

@@ -6,25 +6,30 @@
     The open-source Agentic LLM Vulnerability Scanner
     <br />
     <br />
-
-<p>
-<img alt="GitHub Contributors" src="https://img.shields.io/github/contributors/msoedov/agentic_security" />
-<img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/msoedov/agentic_security" />
-<img alt="" src="https://img.shields.io/github/repo-size/msoedov/agentic_security" />
-<img alt="Downloads" src="https://static.pepy.tech/badge/agentic_security" />
-<img alt="GitHub Issues" src="https://img.shields.io/github/issues/msoedov/agentic_security" />
-<img alt="GitHub Pull Requests" src="https://img.shields.io/github/issues-pr/msoedov/agentic_security" />
-<img alt="Github License" src="https://img.shields.io/github/license/msoedov/agentic_security" />
 </p>
-  </p>
+
+<p align="center">
+  <a href="https://github.com/msoedov/agentic_security/commits/main">
+    <img alt="GitHub Last Commit" src="https://img.shields.io/github/last-commit/msoedov/agentic_security?style=for-the-badge&logo=git&labelColor=000000&logoColor=FFFFFF&label=Last Commit&color=6A35FF" />
+  </a>
+  <a href="https://github.com/msoedov/agentic_security">
+    <img alt="GitHub Repo Size" src="https://img.shields.io/github/repo-size/msoedov/agentic_security?style=for-the-badge&logo=database&labelColor=000000&logoColor=FFFFFF&label=Repo Size&color=yellow" />
+  </a>
+  </a>
+  <a href="https://github.com/msoedov/agentic_security/blob/master/LICENSE">
+    <img alt="GitHub License" src="https://img.shields.io/github/license/msoedov/agentic_security?style=for-the-badge&logo=codeigniter&labelColor=000000&logoColor=FFFFFF&label=License&color=FFCC19" />
+  </a>
+  <a href="https://discord.gg/stw3DfZQ"><img alt="Join the community" src="https://img.shields.io/badge/Join%20the%20community-black.svg?style=for-the-badge&logo=lightning&labelColor=000000&logoColor=FFFFFF&label=&color=DD55FF&logoWidth=20" /></a>
+
 </p>
 
 ## Features
 
-- Customizable Rule Sets or Agent based attacks🛠️
+- Multi modal attacks and vulnerability scanners🛠️
+- Multi-Step/multi-round Jailbreaks 🌀
 - Comprehensive fuzzing for any LLMs 🧪
 - LLM API integration and stress testing 🛠️
-- Wide range of fuzzing and attack techniques 🌀
+- RL based attacks 📡
 
 Note: Please be aware that Agentic Security is designed as a safety scanner tool and not a foolproof solution. It cannot guarantee complete protection against all possible threats.
 

agentic_security/app.py

@@ -8,6 +8,7 @@ from .routes import (
     report_router,
     scan_router,
     static_router,
+    telemetry,
 )
 
 # Create the FastAPI app
@@ -26,3 +27,4 @@ app.include_router(scan_router)
 app.include_router(probe_router)
 app.include_router(proxy_router)
 app.include_router(report_router)
+telemetry.setup(app)

agentic_security/config.py

@@ -0,0 +1,122 @@
+import tomli
+from loguru import logger
+
+
+class CfgMixin:
+    config = {}
+    default_path = "agentic_security.toml"
+
+    def get_or_create_config(self) -> bool:
+        if not self.has_local_config():
+            self.generate_default_cfg()
+            return False
+        self.load_config(self.default_path)
+        return True
+
+    def has_local_config(self):
+        try:
+            with open(self.default_path):
+                return True
+        except FileNotFoundError:
+            return False
+
+    @classmethod
+    def load_config(cls, config_path: str):
+        """
+        Load configuration from a TOML file and store it in the class variable.
+
+        Args:
+            config_path (str): Path to the TOML configuration file.
+
+        Raises:
+            FileNotFoundError: If the configuration file is not found.
+            toml.TomlDecodeError: If the configuration file has syntax errors.
+        """
+        try:
+            with open(config_path, "rb") as config_file:
+                cls.config = tomli.load(config_file)
+                logger.info(f"Configuration loaded successfully from {config_path}.")
+        except FileNotFoundError:
+            logger.error(f"Configuration file {config_path} not found.")
+            raise
+        except Exception as e:
+            logger.error(f"Error parsing TOML configuration: {e}")
+            raise
+
+    @classmethod
+    def get_config_value(cls, key: str, default=None):
+        """
+        Retrieve a configuration value by key from the loaded configuration.
+
+        Args:
+            key (str): Dot-separated key path to the configuration value (e.g., 'general.maxBudget').
+            default: Default value if the key is not found.
+
+        Returns:
+            The configuration value if found, otherwise the default value.
+        """
+        keys = key.split(".")
+        value = cls.config
+        for k in keys:
+            if isinstance(value, dict) and k in value:
+                value = value[k]
+            else:
+                return default
+        return value
+
+    def generate_default_cfg(self, host: str = "0.0.0.0", port: int = 8718):
+        # Accept host / port as parameters
+        with open(self.default_path, "w") as f:
+            f.write(
+                """
+[general]
+# General configuration for the security scan
+llmSpec = \"""
+POST http://$HOST:$PORT/v1/self-probe
+Authorization: Bearer XXXXX
+Content-Type: application/json
+
+{
+    "prompt": "<<PROMPT>>"
+}
+\""" # LLM API specification
+maxBudget = 1000000 # Maximum budget for the scan
+max_th = 0.3 # Maximum failure threshold (percentage)
+optimize = false # Enable optimization during scanning
+enableMultiStepAttack = false # Enable multi-step attack simulations
+
+# [modules.LLM-Jailbreak-Classifier]
+# dataset_name = "markush1/LLM-Jailbreak-Classifier"
+
+[modules.aya-23-8B_advbench_jailbreak]
+dataset_name = "simonycl/aya-23-8B_advbench_jailbreak"
+
+
+[modules.AgenticBackend]
+dataset_name = "AgenticBackend"
+[modules.AgenticBackend.opts]
+port = $PORT
+modules = ["encoding"]
+
+
+[thresholds]
+# Threshold settings
+low = 0.15
+medium = 0.3
+high = 0.5
+
+[secrets]
+# Secrets for the security scan from environment variables
+OPENAI_API_KEY = "$OPENAI_API_KEY"
+DEEPSEEK_API_KEY = "$DEEPSEEK_API_KEY"
+
+""".replace(
+                    "$HOST", host
+                ).replace(
+                    "$PORT", str(port)
+                )
+            )
+
+        logger.info(
+            f"Default configuration generated successfully to {self.default_path}."
+        )

agentic_security/core/app.py

@@ -1,3 +1,4 @@
+import os
 from asyncio import Event, Queue
 
 from fastapi import FastAPI
@@ -5,6 +6,7 @@ from fastapi import FastAPI
 tools_inbox: Queue = Queue()
 stop_event: Event = Event()
 current_run: str = {"spec": "", "id": ""}
+_secrets = {}
 
 
 def create_app() -> FastAPI:
@@ -33,3 +35,20 @@ def set_current_run(spec):
     current_run["id"] = hash(id(spec))
     current_run["spec"] = spec
     return current_run
+
+
+def get_secrets():
+    return _secrets
+
+
+def set_secrets(secrets):
+    _secrets.update(secrets)
+    expand_secrets(_secrets)
+    return _secrets
+
+
+def expand_secrets(secrets):
+    for key in secrets:
+        val = secrets[key]
+        if val.startswith("$"):
+            secrets[key] = os.getenv(val.strip("$"))

agentic_security/core/test_app.py

@@ -0,0 +1,27 @@
+import os
+import pytest
+from agentic_security.core.app import expand_secrets
+
+
+@pytest.fixture(autouse=True)
+def setup_env_vars():
+    # Set up environment variables for testing
+    os.environ["TEST_ENV_VAR"] = "test_value"
+
+
+def test_expand_secrets_with_env_var():
+    secrets = {"secret_key": "$TEST_ENV_VAR"}
+    expand_secrets(secrets)
+    assert secrets["secret_key"] == "test_value"
+
+
+def test_expand_secrets_without_env_var():
+    secrets = {"secret_key": "$NON_EXISTENT_VAR"}
+    expand_secrets(secrets)
+    assert secrets["secret_key"] is None
+
+
+def test_expand_secrets_without_dollar_sign():
+    secrets = {"secret_key": "plain_value"}
+    expand_secrets(secrets)
+    assert secrets["secret_key"] == "plain_value"

agentic_security/dependencies.py

@@ -0,0 +1,29 @@
+from agentic_security.config import CfgMixin
+from agentic_security.core.app import set_secrets
+
+
+class InMemorySecrets:
+    def __init__(self):
+        self.secrets = {}
+        self.config = CfgMixin()
+        self.config.get_or_create_config()
+        self.secrets = self.config.config.get("secrets", {})
+        set_secrets(self.secrets)
+
+    def set_secret(self, key: str, value: str):
+        self.secrets[key] = value
+
+    def get_secret(self, key: str) -> str:
+        return self.secrets.get(key, None)
+
+
+# Dependency
+def get_in_memory_secrets() -> InMemorySecrets:
+    return InMemorySecrets()
+
+
+# Example usage in a FastAPI route
+# @app.get("/some-endpoint")
+# async def some_endpoint(secrets: InMemorySecrets = Depends(get_in_memory_secrets)):
+#     # Use secrets here
+#     pass

agentic_security/http_spec.py

@@ -138,6 +138,9 @@ def parse_http_spec(http_spec: str) -> LLMSpec:
     Returns:
         LLMSpec: An object representing the parsed HTTP specification, with attributes for the method, URL, headers, and body.
     """
+    from agentic_security.core.app import get_secrets
+
+    secrets = get_secrets()
 
     # Split the spec by lines
     lines = http_spec.strip().split("\n")
@@ -164,6 +167,11 @@ def parse_http_spec(http_spec: str) -> LLMSpec:
     has_files = "multipart/form-data" in headers.get("Content-Type", "")
     has_image = "<<BASE64_IMAGE>>" in body
     has_audio = "<<BASE64_AUDIO>>" in body
+
+    for key, value in secrets.items():
+        key = key.strip("$")
+        body = body.replace(f"${key}", value)
+
     return LLMSpec(
         method=method,
         url=url,

agentic_security/integrations/__init__.py

@@ -0,0 +1,12 @@
+import asyncio
+from typing import Protocol
+
+
+class IntegrationProto(Protocol):
+    def __init__(
+        self, prompt_groups: list, tools_inbox: asyncio.Queue, opts: dict = {}
+    ):
+        ...
+
+    async def apply(self) -> list:
+        ...

agentic_security/lib.py

@@ -3,13 +3,13 @@ import json
 from datetime import datetime
 
 import colorama
-import tomli
 import tqdm.asyncio
 from loguru import logger
 from rich.console import Console
 from rich.table import Table
 from tabulate import tabulate
 
+from agentic_security.config import CfgMixin  # Importing the configuration mixin
 from agentic_security.models.schemas import Scan
 from agentic_security.probe_data import REGISTRY
 from agentic_security.routes.scan import streaming_response_generator
@@ -23,62 +23,6 @@ YELLOW = colorama.Fore.YELLOW
 BLUE = colorama.Fore.BLUE
 
 
-class CfgMixin:
-    config = {}
-    default_path = "agesec.toml"
-
-    def has_local_config(self):
-        try:
-            with open(self.default_path):
-                return True
-        except FileNotFoundError:
-            return False
-
-    @classmethod
-    def load_config(cls, config_path: str):
-        """
-        Load configuration from a TOML file and store it in the class variable.
-
-        Args:
-            config_path (str): Path to the TOML configuration file.
-
-        Raises:
-            FileNotFoundError: If the configuration file is not found.
-            toml.TomlDecodeError: If the configuration file has syntax errors.
-        """
-        try:
-            with open(config_path, "rb") as config_file:
-                cls.config = tomli.load(config_file)
-                logger.info(f"Configuration loaded successfully from {config_path}.")
-        except FileNotFoundError:
-            logger.error(f"Configuration file {config_path} not found.")
-            raise
-        except Exception as e:
-            logger.error(f"Error parsing TOML configuration: {e}")
-            raise
-
-    @classmethod
-    def get_config_value(cls, key: str, default=None):
-        """
-        Retrieve a configuration value by key from the loaded configuration.
-
-        Args:
-            key (str): Dot-separated key path to the configuration value (e.g., 'general.maxBudget').
-            default: Default value if the key is not found.
-
-        Returns:
-            The configuration value if found, otherwise the default value.
-        """
-        keys = key.split(".")
-        value = cls.config
-        for k in keys:
-            if isinstance(value, dict) and k in value:
-                value = value[k]
-            else:
-                return default
-        return value
-
-
 class AgenticSecurity(CfgMixin):
     @classmethod
     async def async_scan(
@@ -272,59 +216,6 @@ class AgenticSecurity(CfgMixin):
             ),
         )
 
-    def generate_default_cfg(self, host: str = "0.0.0.0", port: int = 8718):
-        # Accept host / port as parameters
-        with open(self.default_path, "w") as f:
-            f.write(
-                """
-[general]
-# General configuration for the security scan
-llmSpec = \"""
-POST http://$HOST:$PORT/v1/self-probe
-Authorization: Bearer XXXXX
-Content-Type: application/json
-
-{
-    "prompt": "<<PROMPT>>"
-}
-\""" # LLM API specification
-maxBudget = 1000000 # Maximum budget for the scan
-max_th = 0.3 # Maximum failure threshold (percentage)
-optimize = false # Enable optimization during scanning
-enableMultiStepAttack = false # Enable multi-step attack simulations
-
-# [modules.LLM-Jailbreak-Classifier]
-# dataset_name = "markush1/LLM-Jailbreak-Classifier"
-
-[modules.aya-23-8B_advbench_jailbreak]
-dataset_name = "simonycl/aya-23-8B_advbench_jailbreak"
-
-
-[modules.AgenticBackend]
-dataset_name = "AgenticBackend"
-[modules.AgenticBackend.opts]
-port = $PORT
-modules = ["encoding"]
-
-
-[thresholds]
-# Threshold settings
-low = 0.15
-medium = 0.3
-high = 0.5
-
-
-""".replace(
-                    "$HOST", host
-                ).replace(
-                    "$PORT", str(port)
-                )
-            )
-
-        logger.info(
-            f"Default configuration generated successfully to {self.default_path}."
-        )
-
     def list_checks(self):
         """
         Print the REGISTRY contents as a table using the rich library.

agentic_security/models/schemas.py

@@ -23,6 +23,18 @@ class Scan(BaseModel):
     enableMultiStepAttack: bool = False
     # MSJ only mode
     probe_datasets: list[dict] = []
+    # Set and managed by the backend
+    secrets: dict[str, str] = {}
+
+    def with_secrets(self, secrets) -> "Scan":
+        match secrets:
+            case dict():
+                self.secrets.update(secrets)
+            case obj if hasattr(obj, "secrets"):
+                self.secrets.update(obj.secrets)
+            case _:
+                raise ValueError("Invalid secrets type")
+        return self
 
 
 class ScanResult(BaseModel):
@@ -32,6 +44,10 @@ class ScanResult(BaseModel):
     progress: float
     status: bool = False
     failureRate: float = 0.0
+    prompt: str = ""
+    model: str = ""
+    refused: bool = False
+    latency: float = 0.0
 
     @classmethod
     def status_msg(cls, msg: str) -> str:
@@ -42,6 +58,10 @@ class ScanResult(BaseModel):
             progress=0,
             failureRate=0,
             status=True,
+            prompt="",
+            model="",
+            refused=False,
+            latency=0,
         ).model_dump_json()
 
 

agentic_security/probe_actor/cost_module.py

@@ -0,0 +1,58 @@
+def calculate_cost(tokens: int, model: str = "deepseek-chat") -> float:
+    """Calculate API cost based on token count and model.
+
+    Args:
+        tokens (int): Number of tokens used
+        model (str): Model name to calculate cost for
+
+    Returns:
+        float: Cost in USD
+    """
+    # API pricing as of 2024-03-01
+    pricing = {
+        "deepseek-chat": {
+            "input": 0.0007 / 1000,  # $0.70 per million input tokens
+            "output": 0.0028 / 1000,  # $2.80 per million output tokens
+        },
+        "gpt-4-turbo": {
+            "input": 0.01 / 1000,  # $10 per million input tokens
+            "output": 0.03 / 1000,  # $30 per million output tokens
+        },
+        "gpt-4": {
+            "input": 0.03 / 1000,  # $30 per million input tokens
+            "output": 0.06 / 1000,  # $60 per million output tokens
+        },
+        "gpt-3.5-turbo": {
+            "input": 0.0015 / 1000,  # $1.50 per million input tokens
+            "output": 0.002 / 1000,  # $2.00 per million output tokens
+        },
+        "claude-3-opus": {
+            "input": 0.015 / 1000,  # $15 per million input tokens
+            "output": 0.075 / 1000,  # $75 per million output tokens
+        },
+        "claude-3-sonnet": {
+            "input": 0.003 / 1000,  # $3 per million input tokens
+            "output": 0.015 / 1000,  # $15 per million output tokens
+        },
+        "claude-3-haiku": {
+            "input": 0.00025 / 1000,  # $0.25 per million input tokens
+            "output": 0.00125 / 1000,  # $1.25 per million output tokens
+        },
+        "mistral-large": {
+            "input": 0.008 / 1000,  # $8 per million input tokens
+            "output": 0.024 / 1000,  # $24 per million output tokens
+        },
+        "mixtral-8x7b": {
+            "input": 0.002 / 1000,  # $2 per million input tokens
+            "output": 0.006 / 1000,  # $6 per million output tokens
+        },
+    }
+
+    if model not in pricing:
+        raise ValueError(f"Unknown model: {model}")
+
+    # For now, assume 1:1 input/output ratio
+    input_cost = tokens * pricing[model]["input"]
+    output_cost = tokens * pricing[model]["output"]
+
+    return round(input_cost + output_cost, 4)

agentic_security/probe_actor/fuzzer.py

@@ -1,5 +1,6 @@
 import asyncio
 import random
+import time
 from collections.abc import AsyncGenerator
 
 import httpx
@@ -10,12 +11,15 @@ from skopt.space import Real
 
 from agentic_security.http_spec import Modality
 from agentic_security.models.schemas import Scan, ScanResult
+from agentic_security.probe_actor.cost_module import calculate_cost
 from agentic_security.probe_actor.refusal import refusal_heuristic
 from agentic_security.probe_data import audio_generator, image_generator, msj_data
 from agentic_security.probe_data.data import prepare_prompts
 
 # TODO: full log file
 
+MAX_PROMPT_LENGTH = 2048
+
 
 async def generate_prompts(
     prompts: list[str] | AsyncGenerator,
@@ -38,13 +42,14 @@ def multi_modality_spec(llm_spec):
             return llm_spec
         case _:
             return llm_spec
-        # case _:
-        #     raise NotImplementedError(f"Modality {llm_spec.modality} not supported yet")
 
 
 async def process_prompt(
-    request_factory, prompt, tokens, module_name, refusals, errors
-):
+    request_factory, prompt, tokens, module_name, refusals, errors, outputs
+) -> tuple[int, bool]:
+    """
+    Process a single prompt and update the token count and failure status.
+    """
     try:
         response = await request_factory.fn(prompt=prompt)
         if response.status_code == 422:
@@ -53,18 +58,18 @@ async def process_prompt(
             return tokens, True
 
         if response.status_code >= 400:
-            raise httpx.HTTPStatusError(
-                f"HTTP {response.status_code} {response.content=}",
-                request=response.request,
-                response=response,
-            )
+            logger.error(f"HTTP {response.status_code} {response.content=}")
+            errors.append((module_name, prompt, response.status_code, response.text))
+            return tokens, True
         response_text = response.text
         tokens += len(response_text.split())
 
-        if refusal_heuristic(response.json()):
+        refused = refusal_heuristic(response.json())
+        if refused:
             refusals.append((module_name, prompt, response.status_code, response_text))
-            return tokens, True
-        return tokens, False
+
+        outputs.append((module_name, prompt, response_text, refused))
+        return tokens, refused
 
     except httpx.RequestError as exc:
         logger.error(f"Request error: {exc}")
@@ -79,6 +84,7 @@ async def perform_single_shot_scan(
     tools_inbox=None,
     optimize=False,
     stop_event: asyncio.Event = None,
+    secrets: dict[str, str] = {},
 ) -> AsyncGenerator[str, None]:
     """Perform a standard security scan."""
     max_budget = max_budget * 100_000_000
@@ -96,6 +102,7 @@ async def perform_single_shot_scan(
 
         errors = []
         refusals = []
+        outputs = []
         total_prompts = sum(len(m.prompts) for m in prompt_modules if not m.lazy)
         processed_prompts = 0
 
@@ -129,6 +136,7 @@ async def perform_single_shot_scan(
                     100 * processed_prompts / total_prompts if total_prompts else 0
                 )
                 total_tokens -= tokens
+                start = time.time()
                 tokens, failed = await process_prompt(
                     request_factory,
                     prompt,
@@ -136,14 +144,23 @@ async def perform_single_shot_scan(
                     module.dataset_name,
                     refusals,
                     errors,
+                    outputs,
                 )
+                end = time.time()
                 total_tokens += tokens
                 # logger.debug(f"Trying prompt: {prompt}, {failed=}")
                 if failed:
                     module_failures += 1
                 failure_rate = module_failures / max(processed_prompts, 1)
                 failure_rates.append(failure_rate)
-                cost = round(tokens * 1.5 / 1000_000, 2)
+                cost = calculate_cost(tokens)
+
+                # TODO: improve this cond
+                last_output = outputs[-1] if outputs else None
+                if last_output and last_output[1] == prompt:
+                    response_text = last_output[2]
+                else:
+                    response_text = ""
 
                 yield ScanResult(
                     module=module.dataset_name,
@@ -151,6 +168,9 @@ async def perform_single_shot_scan(
                     cost=cost,
                     progress=round(progress, 2),
                     failureRate=round(failure_rate * 100, 2),
+                    prompt=prompt[:MAX_PROMPT_LENGTH],
+                    latency=end - start,
+                    model=response_text,
                 ).model_dump_json()
 
                 if optimize and len(failure_rates) >= 5:
@@ -184,7 +204,9 @@ async def perform_single_shot_scan(
     except Exception as e:
         logger.exception("Scan failed")
         yield ScanResult.status_msg(f"Scan failed: {str(e)}")
-        raise e
+        # raise e
+    finally:
+        yield ScanResult.status_msg("Scan completed.")
 
 
 async def perform_many_shot_scan(
@@ -197,6 +219,7 @@ async def perform_many_shot_scan(
     stop_event: asyncio.Event = None,
     probe_frequency: float = 0.2,
     max_ctx_length: int = 10_000,
+    secrets: dict[str, str] = {},
 ) -> AsyncGenerator[str, None]:
     """Perform a multi-step security scan with probe injection."""
     request_factory = multi_modality_spec(request_factory)
@@ -214,6 +237,7 @@ async def perform_many_shot_scan(
 
         errors = []
         refusals = []
+        outputs = []
         total_prompts = sum(len(m.prompts) for m in prompt_modules if not m.lazy)
         processed_prompts = 0
 
@@ -265,6 +289,7 @@ async def perform_many_shot_scan(
                         module.dataset_name,
                         refusals,
                         errors,
+                        outputs,
                     )
                     if failed:
                         module_failures += 1
@@ -274,7 +299,7 @@ async def perform_many_shot_scan(
 
                 failure_rate = module_failures / max(processed_prompts, 1)
                 failure_rates.append(failure_rate)
-                cost = round(tokens * 1.5 / 1000_000, 2)
+                cost = calculate_cost(tokens)
 
                 yield ScanResult(
                     module=module.dataset_name,
@@ -282,6 +307,7 @@ async def perform_many_shot_scan(
                     cost=cost,
                     progress=round(progress, 2),
                     failureRate=round(failure_rate * 100, 2),
+                    prompt=prompt[:MAX_PROMPT_LENGTH],
                 ).model_dump_json()
 
                 if optimize and len(failure_rates) >= 5:
@@ -322,6 +348,7 @@ def scan_router(
             tools_inbox=tools_inbox,
             optimize=scan_parameters.optimize,
             stop_event=stop_event,
+            secrets=scan_parameters.secrets,
         )
     else:
         return perform_single_shot_scan(
@@ -331,4 +358,5 @@ def scan_router(
             tools_inbox=tools_inbox,
             optimize=scan_parameters.optimize,
             stop_event=stop_event,
+            secrets=scan_parameters.secrets,
         )

agentic_security/probe_actor/operator.py

@@ -1,9 +1,16 @@
 import asyncio
+import logging
 from typing import Any
 
+import httpx
+from httpx import LLMSpec
 from pydantic import BaseModel, Field
 from pydantic_ai import Agent, RunContext
 
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 
 class AgentSpecification(BaseModel):
     name: str | None = Field(None, description="Name of the LLM/agent")
@@ -13,9 +20,9 @@ class AgentSpecification(BaseModel):
     configuration: dict[str, Any] | None = Field(
         None, description="Configuration settings"
     )
+    endpoint: str | None = Field(None, description="Endpoint URL of the deployed agent")
 
 
-# Define the OperatorToolBox class
 class OperatorToolBox:
     def __init__(self, spec: AgentSpecification, datasets: list[dict[str, Any]]):
         self.spec = spec
@@ -29,7 +36,6 @@ class OperatorToolBox:
         return self.datasets
 
     def validate(self) -> bool:
-        # Validate the tool box based on the specification
         if not self.spec.name or not self.spec.version:
             self.failures.append("Invalid specification: Name or version is missing.")
             return False
@@ -39,28 +45,70 @@ class OperatorToolBox:
         return True
 
     def stop(self) -> None:
-        # Stop the tool box
-        print("Stopping the toolbox...")
+        logger.info("Stopping the toolbox...")
 
     def run(self) -> None:
-        # Run the tool box
-        print("Running the toolbox...")
+        logger.info("Running the toolbox...")
 
     def get_results(self) -> list[dict[str, Any]]:
-        # Get the results
         return self.datasets
 
     def get_failures(self) -> list[str]:
-        # Handle failure
         return self.failures
 
     def run_operation(self, operation: str) -> str:
-        # Run an operation based on the specification
         if operation not in ["dataset1", "dataset2", "dataset3"]:
             self.failures.append(f"Operation '{operation}' failed: Dataset not found.")
             return f"Operation '{operation}' failed: Dataset not found."
         return f"Operation '{operation}' executed successfully."
 
+    async def test(self, description: str, sample_test: dict[str, Any]) -> str:
+        agent = Agent(
+            "openai:gpt-4o",
+            result_type=LLMSpec,
+            system_prompt="Extract the LLM specification from the input",
+        )
+
+        async with agent.run_stream(description) as result:
+            async for spec in result.stream():
+                self.spec.endpoint = spec.url
+
+                # Verify access to the endpoint
+                async with httpx.AsyncClient() as client:
+                    try:
+                        access_response = await client.get(spec.url)
+                        access_response.raise_for_status()
+                    except httpx.HTTPStatusError as e:
+                        self.failures.append(f"HTTP error occurred: {e}")
+                        logger.error(f"Access verification failed: {e}")
+                        return f"Access verification failed: {e}"
+                    except Exception as e:
+                        self.failures.append(f"An error occurred: {e}")
+                        logger.error(f"Access verification failed: {e}")
+                        return f"Access verification failed: {e}"
+
+                # Run the sample test
+                try:
+                    test_response = await client.post(
+                        f"{spec.url}/test", json=sample_test
+                    )
+                    test_response.raise_for_status()
+                    response_data = test_response.json()
+                    if "choices" in response_data and len(response_data["choices"]) > 0:
+                        return f"Testing agent at {spec.url} succeeded: {response_data}"
+                    else:
+                        self.failures.append("Invalid response format")
+                        logger.error("Sample test failed: Invalid response format")
+                        return "Sample test failed: Invalid response format"
+                except httpx.HTTPStatusError as e:
+                    self.failures.append(f"HTTP error occurred: {e}")
+                    logger.error(f"Sample test failed: {e}")
+                    return f"Sample test failed: {e}"
+                except Exception as e:
+                    self.failures.append(f"An error occurred: {e}")
+                    logger.error(f"Sample test failed: {e}")
+                    return f"Sample test failed: {e}"
+
 
 # Initialize OperatorToolBox with AgentSpecification
 spec = AgentSpecification(
@@ -71,24 +119,19 @@ spec = AgentSpecification(
     configuration={"max_tokens": 100},
 )
 
-# dataset_manager_agent.py
-
-
-# Initialize OperatorToolBox
 toolbox = OperatorToolBox(spec=spec, datasets=["dataset1", "dataset2", "dataset3"])
 
 # Define the agent with OperatorToolBox as its dependency
 dataset_manager_agent = Agent(
     model="gpt-4",
     deps_type=OperatorToolBox,
-    result_type=str,  # The agent will return string results
+    result_type=str,
     system_prompt="You can validate the toolbox, run operations, and retrieve results or failures.",
 )
 
 
 @dataset_manager_agent.tool
 async def validate_toolbox(ctx: RunContext[OperatorToolBox]) -> str:
-    """Validate the OperatorToolBox."""
     is_valid = ctx.deps.validate()
     if is_valid:
         return "ToolBox validation successful."
@@ -98,14 +141,12 @@ async def validate_toolbox(ctx: RunContext[OperatorToolBox]) -> str:
 
 @dataset_manager_agent.tool
 async def execute_operation(ctx: RunContext[OperatorToolBox], operation: str) -> str:
-    """Execute an operation on a dataset."""
     result = ctx.deps.run_operation(operation)
     return result
 
 
 @dataset_manager_agent.tool
 async def retrieve_results(ctx: RunContext[OperatorToolBox]) -> str:
-    """Retrieve the results of operations."""
     results = ctx.deps.get_results()
     if results:
         formatted_results = "\n".join([f"{op}: {res}" for op, res in results.items()])
@@ -116,7 +157,6 @@ async def retrieve_results(ctx: RunContext[OperatorToolBox]) -> str:
 
 @dataset_manager_agent.tool
 async def retrieve_failures(ctx: RunContext[OperatorToolBox]) -> str:
-    """Retrieve the list of failures."""
     failures = ctx.deps.get_failures()
     if failures:
         formatted_failures = "\n".join(failures)
@@ -125,6 +165,14 @@ async def retrieve_failures(ctx: RunContext[OperatorToolBox]) -> str:
         return "No failures recorded."
 
 
+@dataset_manager_agent.tool
+async def test_agent(
+    ctx: RunContext[OperatorToolBox], description: str, sample_test: dict[str, Any]
+) -> str:
+    result = await ctx.deps.test(description, sample_test)
+    return result
+
+
 # Synchronous run example
 def run_dataset_manager_agent_sync():
     prompts = [
@@ -133,10 +181,18 @@ def run_dataset_manager_agent_sync():
         "Execute operation on 'dataset4'.",  # This should fail
         "Retrieve the results.",
         "Retrieve any failures.",
+        "Test my openAI compatible agent deployed at localhost:3000",
     ]
 
+    sample_test = {"prompt": "Hello, how are you?", "max_tokens": 5}
+
     for prompt in prompts:
-        result = dataset_manager_agent.run_sync(prompt, deps=toolbox)
+        if "Test my" in prompt:
+            result = dataset_manager_agent.run_sync(
+                prompt, deps=toolbox, sample_test=sample_test
+            )
+        else:
+            result = dataset_manager_agent.run_sync(prompt, deps=toolbox)
         print(f"Prompt: {prompt}")
         print(f"Response: {result.data}\n")
 
@@ -149,10 +205,18 @@ async def run_dataset_manager_agent_async():
         "Execute operation on 'dataset4'.",  # This should fail
         "Retrieve the results.",
         "Retrieve any failures.",
+        "Test my openAI compatible agent deployed at localhost:3000",
     ]
 
+    sample_test = {"prompt": "Hello, how are you?", "max_tokens": 5}
+
     for prompt in prompts:
-        result = await dataset_manager_agent.run(prompt, deps=toolbox)
+        if "Test my" in prompt:
+            result = await dataset_manager_agent.run(
+                prompt, deps=toolbox, sample_test=sample_test
+            )
+        else:
+            result = await dataset_manager_agent.run(prompt, deps=toolbox)
         print(f"Prompt: {prompt}")
         print(f"Response: {result.data}\n")
 

agentic_security/probe_actor/test_fuzzer.py

@@ -209,6 +209,7 @@ class TestProcessPrompt(unittest.IsolatedAsyncioTestCase):
             module_name="module_a",
             refusals=[],
             errors=[],
+            outputs=[],
         )
 
         self.assertEqual(tokens, 3)  # Tokens from "Valid response text"
@@ -226,6 +227,7 @@ class TestProcessPrompt(unittest.IsolatedAsyncioTestCase):
         )
 
         refusals = []
+        outputs = []
         tokens, refusal = await process_prompt(
             request_factory=mock_request_factory,
             prompt="test prompt",
@@ -233,6 +235,7 @@ class TestProcessPrompt(unittest.IsolatedAsyncioTestCase):
             module_name="module_a",
             refusals=refusals,
             errors=[],
+            outputs=outputs,
         )
 
         self.assertEqual(tokens, 3)  # Tokens from "Response indicating refusal"
@@ -250,15 +253,15 @@ class TestProcessPrompt(unittest.IsolatedAsyncioTestCase):
         )
 
         refusals = []
-        with self.assertRaises(httpx.HTTPStatusError):
-            await process_prompt(
-                request_factory=mock_request_factory,
-                prompt="test prompt",
-                tokens=0,
-                module_name="module_a",
-                refusals=refusals,
-                errors=[],
-            )
+        await process_prompt(
+            request_factory=mock_request_factory,
+            prompt="test prompt",
+            tokens=0,
+            module_name="module_a",
+            refusals=refusals,
+            errors=[],
+            outputs=[],
+        )
 
     async def test_request_error(self):
         mock_request_factory = Mock()
@@ -274,6 +277,7 @@ class TestProcessPrompt(unittest.IsolatedAsyncioTestCase):
             module_name="module_a",
             refusals=[],
             errors=errors,
+            outputs=[],
         )
 
         self.assertEqual(tokens, 0)

agentic_security/probe_data/audio_generator.py

@@ -52,11 +52,37 @@ def generate_audio_mac_wav(prompt: str) -> bytes:
     return audio_bytes
 
 
+def generate_audio_cross_platform(prompt: str) -> bytes:
+    """
+    Generate an audio file from the provided prompt using gTTS for cross-platform support.
+
+    Parameters:
+        prompt (str): Text to convert into audio.
+
+    Returns:
+        bytes: The audio data in MP3 format.
+    """
+    from gtts import gTTS  # Import gTTS for cross-platform support
+
+    tts = gTTS(text=prompt, lang="en")
+    temp_mp3_path = f"temp_audio_{uuid.uuid4().hex}.mp3"
+    tts.save(temp_mp3_path)
+
+    try:
+        with open(temp_mp3_path, "rb") as f:
+            audio_bytes = f.read()
+    finally:
+        if os.path.exists(temp_mp3_path):
+            os.remove(temp_mp3_path)
+
+    return audio_bytes
+
+
 @cache_to_disk()
 def generate_audioform(prompt: str) -> bytes:
     """
     Generate an audio file from the provided prompt in WAV format.
-    Uses macOS 'say' command if the operating system is macOS.
+    Uses macOS 'say' command if the operating system is macOS, otherwise uses gTTS.
 
     Parameters:
         prompt (str): Text to convert into audio.
@@ -67,9 +93,11 @@ def generate_audioform(prompt: str) -> bytes:
     current_os = platform.system()
     if current_os == "Darwin":  # macOS
         return generate_audio_mac_wav(prompt)
+    elif current_os in ["Windows", "Linux"]:
+        return generate_audio_cross_platform(prompt)
     else:
         raise NotImplementedError(
-            "Audio generation is only supported on macOS for now."
+            "Audio generation is only supported on macOS, Windows, and Linux for now."
         )
 
 

agentic_security/probe_data/image_generator.py

@@ -38,12 +38,13 @@ def generate_image_dataset(
 
 
 @cache_to_disk()
-def generate_image(prompt: str) -> bytes:
+def generate_image(prompt: str, variant: int = 0) -> bytes:
     """
     Generate an image based on the provided prompt and return it as bytes.
 
     Parameters:
         prompt (str): Text to display on the generated image.
+        variant (int): The variant style of the image.
 
     Returns:
         bytes: The image data in JPG format.
@@ -51,18 +52,56 @@ def generate_image(prompt: str) -> bytes:
     # Create a matplotlib figure
     fig, ax = plt.subplots(figsize=(6, 4))
 
-    # Customize the plot (background color, text, etc.)
-    ax.set_facecolor("lightblue")
-    ax.text(
-        0.5,
-        0.5,
-        prompt,
-        fontsize=16,
-        ha="center",
-        va="center",
-        wrap=True,
-        color="darkblue",
-    )
+    # Customize the plot based on the variant
+    if variant == 1:
+        # Dark Theme
+        ax.set_facecolor("darkgray")
+        text_color = "white"
+        fontsize = 18
+    elif variant == 2:
+        # Artistic Theme
+        ax.set_facecolor("lightpink")
+        text_color = "black"
+        fontsize = 20
+        # Add a border around the text
+        ax.text(
+            0.5,
+            0.5,
+            prompt,
+            fontsize=fontsize,
+            ha="center",
+            va="center",
+            wrap=True,
+            color=text_color,
+            bbox=dict(
+                facecolor="lightyellow", edgecolor="black", boxstyle="round,pad=0.5"
+            ),
+        )
+    elif variant == 3:
+        # Minimalist Theme
+        ax.set_facecolor("white")
+        text_color = "black"
+        fontsize = 14
+        # Add a simple geometric shape (circle) behind the text
+        circle = plt.Circle((0.5, 0.5), 0.3, color="lightblue", fill=True)
+        ax.add_artist(circle)
+    else:
+        # Default Theme
+        ax.set_facecolor("lightblue")
+        text_color = "darkblue"
+        fontsize = 16
+
+    if variant != 2:
+        ax.text(
+            0.5,
+            0.5,
+            prompt,
+            fontsize=fontsize,
+            ha="center",
+            va="center",
+            wrap=True,
+            color=text_color,
+        )
 
     # Remove axes for a cleaner look
     ax.axis("off")

agentic_security/probe_data/test_audio_generator.py

@@ -3,6 +3,7 @@ import platform
 import pytest
 
 from agentic_security.probe_data.audio_generator import (
+    generate_audio_cross_platform,
     generate_audio_mac_wav,
     generate_audioform,
 )
@@ -24,6 +25,13 @@ def test_generate_audioform_mac():
         audio_bytes = generate_audioform(prompt)
         assert isinstance(audio_bytes, bytes)
         assert len(audio_bytes) > 0
+
+
+def test_generate_audio_cross_platform():
+    if platform.system() in ["Windows", "Linux"]:
+        prompt = "This is a cross-platform test."
+        audio_bytes = generate_audio_cross_platform(prompt)
+        assert isinstance(audio_bytes, bytes)
+        assert len(audio_bytes) > 0
     else:
-        with pytest.raises(NotImplementedError):
-            generate_audioform("This should raise an error on non-macOS systems.")
+        pytest.skip("Test is only applicable on Windows and Linux.")

agentic_security/probe_data/test_image_generator.py

@@ -1,5 +1,7 @@
 from unittest.mock import patch
 
+import pytest
+
 from agentic_security.probe_data.image_generator import (
     generate_image,
     generate_image_dataset,
@@ -7,9 +9,10 @@ from agentic_security.probe_data.image_generator import (
 from agentic_security.probe_data.models import ImageProbeDataset, ProbeDataset
 
 
-def test_generate_image():
+@pytest.mark.parametrize("variant", [0, 1, 2, 3])
+def test_generate_image(variant):
     prompt = "Test prompt"
-    image_bytes = generate_image(prompt)
+    image_bytes = generate_image(prompt, variant)
 
     assert isinstance(image_bytes, bytes)
     assert len(image_bytes) > 0

agentic_security/routes/scan.py

@@ -1,9 +1,18 @@
 from datetime import datetime
 
-from fastapi import APIRouter, BackgroundTasks, HTTPException
+from fastapi import (
+    APIRouter,
+    BackgroundTasks,
+    Depends,
+    File,
+    HTTPException,
+    Query,
+    UploadFile,
+)
 from fastapi.responses import StreamingResponse
 
 from ..core.app import get_stop_event, get_tools_inbox, set_current_run
+from ..dependencies import InMemorySecrets, get_in_memory_secrets
 from ..http_spec import LLMSpec
 from ..models.schemas import LLMInfo, Scan
 from ..probe_actor import fuzzer
@@ -12,7 +21,9 @@ router = APIRouter()
 
 
 @router.post("/verify")
-async def verify(info: LLMInfo):
+async def verify(
+    info: LLMInfo, secrets: InMemorySecrets = Depends(get_in_memory_secrets)
+):
     spec = LLMSpec.from_string(info.spec)
     r = await spec.verify()
     if r.status_code >= 400:
@@ -42,7 +53,12 @@ def streaming_response_generator(scan_parameters: Scan):
 
 
 @router.post("/scan")
-async def scan(scan_parameters: Scan, background_tasks: BackgroundTasks):
+async def scan(
+    scan_parameters: Scan,
+    background_tasks: BackgroundTasks,
+    secrets: InMemorySecrets = Depends(get_in_memory_secrets),
+):
+    scan_parameters.with_secrets(secrets)
     return StreamingResponse(
         streaming_response_generator(scan_parameters), media_type="application/json"
     )
@@ -52,3 +68,29 @@ async def scan(scan_parameters: Scan, background_tasks: BackgroundTasks):
 async def stop_scan():
     get_stop_event().set()
     return {"status": "Scan stopped"}
+
+
+@router.post("/scan-csv")
+async def scan_csv(
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(...),
+    llmSpec: UploadFile = File(...),
+    optimize: bool = Query(False),
+    maxBudget: int = Query(10_000),
+    enableMultiStepAttack: bool = Query(False),
+    secrets: InMemorySecrets = Depends(get_in_memory_secrets),
+):
+    # TODO: content dataset to fuzzer
+    content = await file.read()  # noqa
+    llm_spec = await llmSpec.read()
+
+    scan_parameters = Scan(
+        llmSpec=llm_spec,
+        optimize=optimize,
+        maxBudget=1000,
+        enableMultiStepAttack=enableMultiStepAttack,
+    )
+    scan_parameters.with_secrets(secrets)
+    return StreamingResponse(
+        streaming_response_generator(scan_parameters), media_type="application/json"
+    )

agentic_security/routes/static.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 
+import requests
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import FileResponse, HTMLResponse
 from fastapi.templating import Jinja2Templates
@@ -10,6 +11,7 @@ from ..models.schemas import Settings
 
 router = APIRouter()
 STATIC_DIR = Path(__file__).parent.parent / "static"
+ICONS_DIR = STATIC_DIR / "icons"
 
 # Configure templates with custom delimiters to avoid conflicts
 templates = Jinja2Templates(directory=str(STATIC_DIR))
@@ -28,6 +30,8 @@ CONTENT_TYPES = {
     ".ico": "image/x-icon",
     ".html": "text/html",
     ".css": "text/css",
+    ".svg": "image/svg+xml",
+    ".png": "image/png",
 }
 
 
@@ -88,3 +92,94 @@ async def telemetry_js() -> FileResponse:
 async def favicon() -> FileResponse:
     """Serve the favicon."""
     return get_static_file(STATIC_DIR / "favicon.ico")
+
+
+@router.get("/icons/{icon_name}")
+async def serve_icon(icon_name: str) -> FileResponse:
+    """Serve an icon from the icons directory."""
+    icon_path = ICONS_DIR / icon_name
+    if not icon_path.exists():
+        # Fetch the icon from the external URL and cache it
+        url = f"https://registry.npmmirror.com/@lobehub/icons-static-png/latest/files/dark/{icon_name}"
+        response = requests.get(url)
+        if response.status_code == 200:
+            icon_path.write_bytes(response.content)
+        else:
+            raise HTTPException(status_code=404, detail="Icon not found")
+
+    return get_static_file(icon_path, content_type="image/png")
+
+
+# New endpoints for proxying external resources
+@router.get("/cdn/tailwindcss.js")
+async def proxy_tailwindcss() -> FileResponse:
+    """Proxy the Tailwind CSS script."""
+    return proxy_external_resource(
+        "https://cdn.tailwindcss.com",
+        STATIC_DIR / "tailwindcss.js",
+        "application/javascript",
+    )
+
+
+@router.get("/cdn/vue.js")
+async def proxy_vue() -> FileResponse:
+    """Proxy the Vue.js script."""
+    return proxy_external_resource(
+        "https://unpkg.com/vue@2.6.12/dist/vue.js",
+        STATIC_DIR / "vue.js",
+        "application/javascript",
+    )
+
+
+@router.get("/cdn/lucide.js")
+async def proxy_lucide() -> FileResponse:
+    """Proxy the Lucide.js script."""
+    return proxy_external_resource(
+        "https://unpkg.com/lucide@latest/dist/umd/lucide.js",
+        STATIC_DIR / "lucide.js",
+        "application/javascript",
+    )
+
+
+@router.get("/cdn/technopollas.css")
+async def proxy_technopollas() -> FileResponse:
+    """Proxy the Technopollas font stylesheet."""
+    return proxy_external_resource(
+        "https://fonts.cdnfonts.com/css/technopollas",
+        STATIC_DIR / "technopollas.css",
+        "text/css",
+    )
+
+
+@router.get("/cdn/inter.css")
+async def proxy_inter() -> FileResponse:
+    """Proxy the Inter font stylesheet."""
+    return proxy_external_resource(
+        "https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&display=swap",
+        STATIC_DIR / "inter.css",
+        "text/css",
+    )
+
+
+def proxy_external_resource(
+    url: str, local_path: Path, content_type: str
+) -> FileResponse:
+    """
+    Fetch and cache an external resource, then serve it locally.
+
+    Args:
+        url: The URL of the external resource
+        local_path: The local path to cache the resource
+        content_type: The content type of the resource
+
+    Returns:
+        FileResponse with the cached resource
+    """
+    if not local_path.exists():
+        response = requests.get(url)
+        if response.status_code == 200:
+            local_path.write_bytes(response.content)
+        else:
+            raise HTTPException(status_code=404, detail="Resource not found")
+
+    return get_static_file(local_path, content_type=content_type)

agentic_security/routes/telemetry.py

@@ -0,0 +1,27 @@
+import sentry_sdk
+from loguru import logger
+from sentry_sdk.integrations.logging import ignore_logger
+
+from ..models.schemas import Settings
+
+
+def setup(app):
+    if Settings.DISABLE_TELEMETRY:
+        return
+    sentry_sdk.init(
+        dsn="https://b5c59f7e5ab86d73518222ddb40807c9@o4508851738247168.ingest.de.sentry.io/4508851740541008",
+        # Add data like request headers and IP for users,
+        # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
+        send_default_pii=True,
+        # Set traces_sample_rate to 1.0 to capture 100%
+        # of transactions for tracing.
+        traces_sample_rate=1.0,
+        _experiments={
+            # Set continuous_profiling_auto_start to True
+            # to automatically start the profiler on when
+            # possible.
+            "continuous_profiling_auto_start": True,
+        },
+    )
+    ignore_logger("logging.error")
+    ignore_logger(logger.error)

agentic_security/routes/test_csv.py

@@ -0,0 +1,22 @@
+from fastapi.testclient import TestClient
+
+import agentic_security.test_spec_assets as test_spec_assets
+from agentic_security.routes.scan import router
+
+client = TestClient(router)
+
+
+def test_upload_csv_and_run():
+    # Create a sample CSV content
+    csv_content = "id,prompt\nspec1,value1\nspec2,value3"
+    # Send a POST request to the /upload-csv endpoint
+    response = client.post(
+        "/scan-csv?optimize=false&enableMultiStepAttack=false&maxBudget=1000",
+        files={
+            "file": ("test.csv", csv_content, "text/csv"),
+            "llmSpec": ("spec.txt", test_spec_assets.SAMPLE_SPEC, "text/plain"),
+        },
+    )
+
+    assert response.status_code == 200
+    assert "Scan completed." in response.text

agentic_security/static/base.js

@@ -1,13 +1,13 @@
 
-let URL = window.location.href;
-if (URL.endsWith('/')) {
-  URL = URL.slice(0, -1);
+let SELF_URL = window.location.href;
+if (SELF_URL.endsWith('/')) {
+  SELF_URL = SELF_URL.slice(0, -1);
 }
-URL = URL.replace('/#', '');
+SELF_URL = SELF_URL.replace('/#', '');
 
 // Vue application
 let LLM_SPECS = [
-  `POST ${URL}/v1/self-probe
+  `POST ${SELF_URL}/v1/self-probe
 Authorization: Bearer XXXXX
 Content-Type: application/json
 
@@ -79,7 +79,7 @@ Content-Type: application/json
 ]
 }
 `,
-  `POST ${URL}/v1/self-probe-image
+  `POST ${SELF_URL}/v1/self-probe-image
 Authorization: Bearer XXXXX
 Content-Type: application/json
 
@@ -101,7 +101,7 @@ Content-Type: application/json
     }
 ]
 `,
-  `POST ${URL}/v1/self-probe-file
+  `POST ${SELF_URL}/v1/self-probe-file
 Authorization: Bearer $GROQ_API_KEY
 Content-Type: multipart/form-data
 
@@ -175,25 +175,23 @@ Content-Type: application/json
 
 ]
 
+let fallbackIcon = '/icons/myshell.png';
 
 let LLM_CONFIGS = [
-  { name: 'Custom API', prompts: 40000, customInstructions: 'Requires api spec' },
-  { name: 'Open AI', prompts: 24000 },
-  { name: 'Deepseek v1', prompts: 24000 },
-  { name: 'Replicate', prompts: 40000 },
-  { name: 'Groq', prompts: 40000 },
-  { name: 'Together.ai', prompts: 40000 },
-  { name: 'Custom API Image', prompts: 40000, customInstructions: 'Requires api spec', modality: 'Image' },
-  { name: 'Custom API Files', prompts: 40000, customInstructions: 'Requires api spec', modality: 'Files' },
-  { name: 'Gemini', prompts: 40000 },
-  { name: 'Claude', prompts: 40000 },
-  { name: 'Cohere', prompts: 40000 },
-  { name: 'Azure OpenAI', prompts: 40000 },
-  { name: 'assemblyai', prompts: 40000 },
-
-
-]
-
+  { name: 'Custom API', prompts: 40000, customInstructions: 'Requires api spec', logo: fallbackIcon },
+  { name: 'Open AI', prompts: 24000, logo: '/icons/openai.png' },
+  { name: 'Deepseek v1', prompts: 24000, logo: '/icons/deepseek.png' },
+  { name: 'Replicate', prompts: 40000, logo: '/icons/replicate.png' },
+  { name: 'Groq', prompts: 40000, logo: '/icons/groq.png' },
+  { name: 'Together.ai', prompts: 40000, logo: '/icons/together.png' },
+  { name: 'Custom API Image', prompts: 40000, customInstructions: 'Requires api spec', modality: 'Image', logo: fallbackIcon },
+  { name: 'Custom API Files', prompts: 40000, customInstructions: 'Requires api spec', modality: 'Files', logo: fallbackIcon },
+  { name: 'Gemini', prompts: 40000, logo: '/icons/gemini.png' },
+  { name: 'Claude', prompts: 40000, logo: '/icons/claude.png' },
+  { name: 'Cohere', prompts: 40000, logo: '/icons/cohere.png' },
+  { name: 'Azure OpenAI', prompts: 40000, logo: '/icons/azureai.png' },
+  { name: 'assemblyai', prompts: 40000, logo: fallbackIcon },
+];
 function has_image(spec) {
   return spec.includes('<<BASE64_IMAGE>>');
 }

agentic_security/static/index.html

@@ -33,8 +33,38 @@
       </header>
       [[% include "partials/concent.html" %]]
 
+          <div class="flex space-x-4 overflow-x-auto scrollbar-hide">
+            <div
+              v-for="(config, index) in configs"
+              :key="index"
+              @click="selectConfig(index)"
+              class="flex-none w-1/2 sm:w-1/3 md:w-1/4 lg:w-1/5 border-2 rounded-lg p-4 flex flex-col items-start transition-all hover:shadow-md cursor-pointer"
+              :class="{
+          'border-dark-accent-green': selectedConfig === index,
+          'border-gray-600': selectedConfig !== index
+        }">
+              <div class="flex items-center font-medium mb-2">
+                <img
+                  v-if="config.logo"
+                  :src="config.logo"
+                  class="w-6 h-6 ml-2 rounded-full"
+                  alt="logo" />
+                <span class="ml-2">{{ config.name }}</span>
+
+              </div>
+
+              <div class="text-sm text-gray-400">
+                {{ config.customInstructions || 'Requires API key' }}
+              </div>
+              <div class="mt-2 text-dark-accent-green font-semibold">
+                {{ config.modality || 'API' }}
+              </div>
+            </div>
+          </div>
+        </section>
+      </main>
       <main class="max-w-6xl mx-auto space-y-8">
-        <section class="bg-dark-card rounded-lg p-6 shadow-lg">
+        <section class="bg-dark-card rounded-lg p-6 shadow-lg" v-show="false">
           <h2 class="text-2xl font-bold mb-4">Select a Config</h2>
 
           <div class="flex space-x-4 overflow-x-auto scrollbar-hide">
@@ -64,7 +94,7 @@
 
             <h2 class="text-2xl font-bold">LLM API Spec</h2>
             <span :class="statusDotClass"
-              class="w-3 h-3 rounded-full mr-2"></span>
+                class="w-3 h-3 rounded-full mr-2"></span>
             <svg :class="{'rotate-180': showLLMSpec}"
               class="w-6 h-6 transition-transform duration-200"
               xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
@@ -74,7 +104,7 @@
             </svg>
           </div>
 
-          <div v-show="showLLMSpec" class="mt-4">
+          <div class="mt-4">
             <label v-if="isFocused" for="llm-spec"
               class="block text-sm font-medium mb-2">
               LLM API Spec, PROMPT variable will be replaced with the testing
@@ -109,6 +139,8 @@
               <strong class="font-bold">></strong>
               <span class="block sm:inline">{{okMsg}}</span>
             </div>
+            <span v-if="latency" class="text-sm text-gray-400 ml-2">Latency: {{latency}}s</span>
+
 
             <!-- Action Buttons -->
             <section class="flex justify-center space-x-4 mt-10">
@@ -388,6 +420,8 @@
           <strong class="font-bold">></strong>
           <span class="block sm:inline">{{okMsg}}</span>
         </div>
+        <span v-if="latency" class="text-sm text-gray-400 ml-2">Latency: {{latency}}s</span>
+
 
         <!-- Action Buttons -->
         <section class="flex justify-center space-x-4">
@@ -437,7 +471,7 @@
                   <th class="p-3">Vulnerability Module</th>
                   <th class="p-3">% Strength</th>
                   <th class="p-3">Number of Tokens</th>
-                  <th class="p-3">Cost (in gpt-3 tokens)</th>
+                  <th class="p-3">Approx Cost (in tokens)</th>
                 </tr>
               </thead>
               <tbody>

agentic_security/static/inter.css

@@ -0,0 +1,21 @@
+@font-face {
+  font-family: 'Inter';
+  font-style: normal;
+  font-weight: 400;
+  font-display: swap;
+  src: url(https://fonts.gstatic.com/s/inter/v18/UcCO3FwrK3iLTeHuS_nVMrMxCp50SjIw2boKoduKmMEVuLyfMZg.ttf) format('truetype');
+}
+@font-face {
+  font-family: 'Inter';
+  font-style: normal;
+  font-weight: 600;
+  font-display: swap;
+  src: url(https://fonts.gstatic.com/s/inter/v18/UcCO3FwrK3iLTeHuS_nVMrMxCp50SjIw2boKoduKmMEVuGKYMZg.ttf) format('truetype');
+}
+@font-face {
+  font-family: 'Inter';
+  font-style: normal;
+  font-weight: 700;
+  font-display: swap;
+  src: url(https://fonts.gstatic.com/s/inter/v18/UcCO3FwrK3iLTeHuS_nVMrMxCp50SjIw2boKoduKmMEVuFuYMZg.ttf) format('truetype');
+}

agentic_security/static/main.js

@@ -4,6 +4,7 @@ var app = new Vue({
         progressWidth: '0%',
         modelSpec: LLM_SPECS[0],
         budget: 50,
+        latency: 0,
         isFocused: false, // Tracks if the textarea is focused
         showParams: false,
         showResetConfirmation: false,
@@ -121,6 +122,7 @@ var app = new Vue({
             const state = {
                 modelSpec: this.modelSpec,
                 budget: this.budget,
+                selectedConfig: this.selectedConfig,
                 dataConfig: this.dataConfig,
                 optimize: this.optimize,
                 enableChartDiagram: this.enableChartDiagram,
@@ -139,6 +141,7 @@ var app = new Vue({
                 this.optimize = state.optimize;
                 this.enableChartDiagram = state.enableChartDiagram;
                 this.enableMultiStepAttack = state.enableMultiStepAttack;
+                this.selectedConfig = state.selectedConfig;
             }
         },
         resetState() {
@@ -190,7 +193,8 @@ var app = new Vue({
             let payload = {
                 spec: this.modelSpec,
             };
-            const response = await fetch(`${URL}/verify`, {
+            let startTime = performance.now(); // Capture start time
+            const response = await fetch(`${SELF_URL}/verify`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
@@ -198,10 +202,14 @@ var app = new Vue({
                 body: JSON.stringify(payload),
             });
             console.log(response);
-            let txt = await response.text();
+            let r = await response.json();
+            let endTime = performance.now(); // Capture end time
+            let latency = endTime - startTime; // Calculate latency in milliseconds
+            latency = latency.toFixed(3) / 1000; // Round to 2 decimal places
+            this.latency = latency;
             if (!response.ok) {
                 this.updateStatusDot(false);
-                this.errorMsg = 'Integration verification failed:' + txt;
+                this.errorMsg = 'Integration verification failed:' + JSON.stringify(r);
             } else {
                 this.errorMsg = '';
                 this.updateStatusDot(true);
@@ -214,7 +222,7 @@ var app = new Vue({
             this.saveStateToLocalStorage();
         },
         loadConfigs: async function () {
-            const response = await fetch(`${URL}/v1/data-config`, {
+            const response = await fetch(`${SELF_URL}/v1/data-config`, {
                 method: 'GET',
                 headers: {
                     'Content-Type': 'application/json',
@@ -286,6 +294,7 @@ var app = new Vue({
                 this.okMsg = `${event.module}`;
                 return
             }
+            this.latency = event.latency.toFixed(3);
             console.log('New event');
             //  { "module": "Module 49", "tokens": 480, "cost": 4.800000000000001, "progress": 9.8 }
             let progress = event.progress;
@@ -321,14 +330,14 @@ var app = new Vue({
             let payload = {
                 table: this.mainTable,
             };
-            const response = await fetch(`${URL}/plot.jpeg`, {
+            const response = await fetch(`${SELF_URL}/plot.jpeg`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
                 },
                 body: JSON.stringify(payload),
             });
-            // Convert image response to a data URL for the <img> src
+            // Convert image response to a data SELF_URL for the <img> src
             const blob = await response.blob();
             const reader = new FileReader();
             reader.readAsDataURL(blob);
@@ -371,7 +380,7 @@ var app = new Vue({
         },
         stopScan: async function () {
             this.scanRunning = false;
-            const response = await fetch(`${URL}/stop`, {
+            const response = await fetch(`${SELF_URL}/stop`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
@@ -387,7 +396,7 @@ var app = new Vue({
                 optimize: this.optimize,
                 enableMultiStepAttack: this.enableMultiStepAttack,
             };
-            const response = await fetch(`${URL}/scan`, {
+            const response = await fetch(`${SELF_URL}/scan`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',

agentic_security/static/partials/footer.html

@@ -6,7 +6,7 @@
             <div>
                 <h3
                     class="text-lg font-semibold text-dark-accent-green mb-4">Home</h3>
-                <p class="text-gray-400">Dedicated to LLM Security, 2024</p>
+                <p class="text-gray-400">Dedicated to LLM Security, 2025</p>
             </div>
 
             <!-- Column 2 -->

agentic_security/static/partials/head.html

@@ -2,12 +2,12 @@
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LLM Vulnerability Scanner</title>
-    <script src="https://cdn.tailwindcss.com"></script>
-    <script src="https://unpkg.com/vue@2.6.12/dist/vue.js"></script>
-    <script src="https://unpkg.com/lucide@latest/dist/umd/lucide.js"></script>
-    <link href="https://fonts.cdnfonts.com/css/technopollas" rel="stylesheet">
+    <script src="/cdn/tailwindcss.js"></script>
+    <script src="/cdn/vue.js"></script>
+    <script src="/cdn/lucide.js"></script>
+    <link href="/cdn/technopollas.css" rel="stylesheet">
     <style>
-      @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&display=swap');
+      @import url('/cdn/inter.css');
     </style>
     <script>
       tailwind.config = {
@@ -19,6 +19,17 @@
               technopollas: ['Technopollas', 'sans-serif'],
             },
             colors: {
+                t1: {
+                bg: '#0D0D0D', // Jet Black
+                card: '#1A1A1A', // Dark Carbon Fiber
+                text: '#FFFFFF',
+                accent: {
+                  green: '#E0A3B6', // Frozen Berry
+                  red: '#1C3F74', // Neptune Blue
+                  orange: '#A5A5A5', // Dolomite Silver
+                  yellow: '#2E4053', // Jet Black
+                },
+              },
               dark: {
                 bg: '#121212',
                 card: '#1E1E1E',
@@ -28,7 +39,44 @@
                   red: '#F44336',
                   orange: '#FF9800',
                   yellow: '#FFEB3B',
+                // bg: '#0D0D0D', // Jet Black
+                // card: '#1A1A1A', // Dark Carbon Fiber
+                // text: '#FFFFFF',
+                // accent: {
+                //   green: '#E0A3B6', // Frozen Berry
+                //   red: '#1C3F74', // Neptune Blue
+                //   orange: '#A5A5A5', // Dolomite Silver
+                //   yellow: '#2E4053', // Jet Black
+
+                  berry: '#E0A3B6', // Frozen Berry
+                  blue: '#1C3F74', // Neptune Blue
+                  silver: '#A5A5A5', // Dolomite Silver
+                  black: '#DAF7A6', // Jet Black
                 },
+                variant1: {
+                    primary: '#E0A3B6', // Frozen Berry
+                    secondary: '#1C3F74', // Neptune Blue
+                    highlight: '#A5A5A5', // Dolomite Silver
+                    dark: '#000000' // Jet Black
+                  },
+                  variant2: {
+                    primary: '#FF5733', // Lava Red
+                    secondary: '#2E4053', // Midnight Blue
+                    highlight: '#C0C0C0', // Platinum Silver
+                    dark: '#121212' // Deep Black
+                  },
+                  variant3: {
+                    primary: '#3D9970', // Racing Green
+                    secondary: '#85144B', // Burgundy Red
+                    highlight: '#AAAAAA', // Light Silver
+                    dark: '#111111' // Matte Black
+                  },
+                  variant4: {
+                    primary: '#FFC300', // Golden Yellow
+                    secondary: '#DAF7A6', // Soft Mint
+                    highlight: '#888888', // Titanium Gray
+                    dark: '#222222' // Charcoal Black
+                  },
               },
             },
             borderRadius: {

agentic_security/static/technopollas.css

@@ -0,0 +1,8 @@
+@font-face {
+    font-family: 'Technopollas';
+    font-style: normal;
+    font-weight: 400;
+    src: local('Technopollas'), url('https://fonts.cdnfonts.com/s/72836/Technopollas.woff') format('woff');
+}
+
+

agentic_security/static/telemetry.js

@@ -2,3 +2,5 @@
 posthog.init('phc_jfYo5xEofW7eJtiU8rLt2Z8jw1E2eW27BxwTJzwRufH', {
     api_host: 'https://us.i.posthog.com', person_profiles: 'identified_only' // or 'always' to create profiles for anonymous users as well
 })
+
+!function (n, e, r, t, o, i, a, c, s) { for (var u = s, f = 0; f < document.scripts.length; f++)if (document.scripts[f].src.indexOf(i) > -1) { u && "no" === document.scripts[f].getAttribute("data-lazy") && (u = !1); break } var p = []; function l(n) { return "e" in n } function d(n) { return "p" in n } function _(n) { return "f" in n } var v = []; function y(n) { u && (l(n) || d(n) || _(n) && n.f.indexOf("capture") > -1 || _(n) && n.f.indexOf("showReportDialog") > -1) && L(), v.push(n) } function h() { y({ e: [].slice.call(arguments) }) } function g(n) { y({ p: n }) } function E() { try { n.SENTRY_SDK_SOURCE = "loader"; var e = n[o], i = e.init; e.init = function (o) { n.removeEventListener(r, h), n.removeEventListener(t, g); var a = c; for (var s in o) Object.prototype.hasOwnProperty.call(o, s) && (a[s] = o[s]); !function (n, e) { var r = n.integrations || []; if (!Array.isArray(r)) return; var t = r.map((function (n) { return n.name })); n.tracesSampleRate && -1 === t.indexOf("BrowserTracing") && (e.browserTracingIntegration ? r.push(e.browserTracingIntegration({ enableInp: !0 })) : e.BrowserTracing && r.push(new e.BrowserTracing)); (n.replaysSessionSampleRate || n.replaysOnErrorSampleRate) && -1 === t.indexOf("Replay") && (e.replayIntegration ? r.push(e.replayIntegration()) : e.Replay && r.push(new e.Replay)); n.integrations = r }(a, e), i(a) }, setTimeout((function () { return function (e) { try { "function" == typeof n.sentryOnLoad && (n.sentryOnLoad(), n.sentryOnLoad = void 0) } catch (n) { console.error("Error while calling `sentryOnLoad` handler:"), console.error(n) } try { for (var r = 0; r < p.length; r++)"function" == typeof p[r] && p[r](); p.splice(0); for (r = 0; r < v.length; r++) { _(i = v[r]) && "init" === i.f && e.init.apply(e, i.a) } m() || e.init(); var t = n.onerror, o = n.onunhandledrejection; for (r = 0; r < v.length; r++) { var i; if (_(i = v[r])) { if ("init" === i.f) continue; e[i.f].apply(e, i.a) } else l(i) && t ? t.apply(n, i.e) : d(i) && o && o.apply(n, [i.p]) } } catch (n) { console.error(n) } }(e) })) } catch (n) { console.error(n) } } var O = !1; function L() { if (!O) { O = !0; var n = e.scripts[0], r = e.createElement("script"); r.src = a, r.crossOrigin = "anonymous", r.addEventListener("load", E, { once: !0, passive: !0 }), n.parentNode.insertBefore(r, n) } } function m() { var e = n.__SENTRY__, r = void 0 !== e && e.version; return r ? !!e[r] : !(void 0 === e || !e.hub || !e.hub.getClient()) } n[o] = n[o] || {}, n[o].onLoad = function (n) { m() ? n() : p.push(n) }, n[o].forceLoad = function () { setTimeout((function () { L() })) }, ["init", "addBreadcrumb", "captureMessage", "captureException", "captureEvent", "configureScope", "withScope", "showReportDialog"].forEach((function (e) { n[o][e] = function () { y({ f: e, a: arguments }) } })), n.addEventListener(r, h), n.addEventListener(t, g), u || setTimeout((function () { L() })) }(window, document, "error", "unhandledrejection", "Sentry", 'a3abb155d8e2fe980880571166594672', 'https://browser.sentry-cdn.com/8.55.0/bundle.tracing.replay.min.js', { "dsn": "https://a3abb155d8e2fe980880571166594672@o4508851738247168.ingest.de.sentry.io/4508851744342096", "tracesSampleRate": 1, "replaysSessionSampleRate": 0.1, "replaysOnErrorSampleRate": 1 }, false);

agentic_security/test_dependencies.py

@@ -0,0 +1,15 @@
+from agentic_security.dependencies import InMemorySecrets, get_in_memory_secrets
+
+
+def test_in_memory_secrets():
+    secrets = InMemorySecrets()
+    secrets.set_secret("api_key", "12345")
+    assert secrets.get_secret("api_key") == "12345"
+    assert secrets.get_secret("non_existent_key") is None
+
+
+def test_get_in_memory_secrets():
+    secrets = get_in_memory_secrets()
+    assert isinstance(secrets, InMemorySecrets)
+    secrets.set_secret("token", "abcde")
+    assert secrets.get_secret("token") == "abcde"

changelog.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Get the last tag
+LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null)
+
+if [ -z "$LAST_TAG" ]; then
+    echo "No tags found. Retrieving all commits."
+    LOG_RANGE="HEAD"
+else
+    echo "Generating changelog from last tag: $LAST_TAG"
+    LOG_RANGE="$LAST_TAG..HEAD"
+fi
+
+# Retrieve commit messages excluding merge commits and format them with author names and stripped email domain as nickname
+CHANGELOG=$(git log --pretty=format:"- %s by %an, @%ae)" --no-merges $LOG_RANGE | sed -E 's/@([^@]+)@([^@]+)\..*/@\1/')
+
+# Output the changelog
+if [ -n "$CHANGELOG" ]; then
+    echo "# Changelog"
+    echo "
+## Changes since $LAST_TAG"
+    echo "$CHANGELOG"
+else
+    echo "No new commits since last tag."
+fi

docs/abstractions.md

@@ -0,0 +1,55 @@
+# Abstractions in Agentic Security
+
+This document outlines the key abstractions used in the Agentic Security project, providing insights into the classes, interfaces, and design patterns that form the backbone of the system.
+
+## Key Abstractions
+
+### AgentSpecification
+
+- **Purpose**: Defines the specification for a language model or agent, including its name, version, description, capabilities, and configuration settings.
+- **Usage**: Used to initialize and configure the `OperatorToolBox` and other components that interact with language models.
+
+### OperatorToolBox
+
+- **Purpose**: Serves as the main class for managing dataset operations, including validation, execution, and result retrieval.
+- **Methods**:
+  - `get_spec()`: Returns the agent specification.
+  - `get_datasets()`: Retrieves the datasets for operations.
+  - `validate()`: Validates the toolbox setup.
+  - `run_operation(operation: str)`: Executes a specified operation.
+
+### DatasetManagerAgent
+
+- **Purpose**: Provides tools for managing and executing operations on datasets through an agent-based approach.
+- **Tools**:
+  - `validate_toolbox`: Validates the `OperatorToolBox`.
+  - `execute_operation`: Executes operations on datasets.
+  - `retrieve_results`: Retrieves operation results.
+  - `retrieve_failures`: Retrieves any failures encountered.
+
+### ProbeDataset
+
+- **Purpose**: Represents a dataset used in security scans, including metadata, prompts, and associated costs.
+- **Methods**:
+  - `metadata_summary()`: Provides a summary of the dataset's metadata.
+
+### Refusal Classifier
+
+- **Purpose**: Analyzes responses from language models to detect potential security vulnerabilities.
+- **Design**: Utilizes predefined rules and machine learning models for classification.
+
+## Design Patterns
+
+### Modular Architecture
+
+- **Description**: The system is designed with a modular architecture, allowing for easy integration of new components and features.
+- **Benefits**: Enhances flexibility, extensibility, and scalability.
+
+### Agent-Based Design
+
+- **Description**: Utilizes an agent-based approach for managing and executing operations on datasets.
+- **Benefits**: Provides a structured framework for interacting with language models and datasets.
+
+## Conclusion
+
+The abstractions in Agentic Security are designed to provide a flexible and extensible framework for managing and executing security scans on language models. This document highlights the key classes, interfaces, and design patterns that contribute to the system's architecture and functionality.

docs/design.md

@@ -0,0 +1,51 @@
+# Design Document
+
+This document provides an overview of the design and architecture of the Agentic Security project. It outlines the key components, their interactions, and the design principles guiding the development of the system.
+
+## Overview
+
+Agentic Security is an open-source LLM vulnerability scanner designed to identify and mitigate potential security threats in language models. It integrates various modules and datasets to perform comprehensive security scans.
+
+## Architecture
+
+The system is built around a modular architecture, allowing for flexibility and extensibility. The core components include:
+
+- **Agentic Security Core**: The main application responsible for orchestrating the security scans and managing interactions with external modules.
+- **Probe Actor**: Handles the execution of fuzzing and attack techniques on language models.
+- **Probe Data**: Manages datasets used for testing and validation, including loading and processing data.
+- **Refusal Classifier**: Analyzes responses from language models to identify potential security issues.
+
+## Key Components
+
+### Agentic Security Core
+
+The core application is responsible for initializing the system, managing configurations, and coordinating the execution of security scans. It provides a command-line interface for users to interact with the system.
+
+### Probe Actor
+
+The Probe Actor module implements various fuzzing and attack techniques. It is designed to test the robustness of language models by simulating different attack scenarios.
+
+### Probe Data
+
+The Probe Data module manages datasets used in security scans. It supports loading data from local files and external sources, providing a flexible framework for testing different scenarios.
+
+### Refusal Classifier
+
+The Refusal Classifier analyzes responses from language models to detect potential security vulnerabilities. It uses predefined rules and machine learning models to classify responses.
+
+## Design Principles
+
+- **Modularity**: The system is designed to be modular, allowing for easy integration of new components and features.
+- **Extensibility**: New modules and datasets can be added to the system without significant changes to the core architecture.
+- **Scalability**: The system is built to handle large datasets and complex security scans efficiently.
+
+## Interaction Flow
+
+1. **Initialization**: The system is initialized with the necessary configurations and datasets.
+1. **Execution**: The Probe Actor executes security scans on the language models using the datasets provided by the Probe Data module.
+1. **Analysis**: The Refusal Classifier analyzes the responses to identify potential security issues.
+1. **Reporting**: Results are compiled and presented to the user, highlighting any vulnerabilities detected.
+
+## Conclusion
+
+The design of Agentic Security emphasizes flexibility, extensibility, and scalability, providing a robust framework for identifying and mitigating security threats in language models. This document serves as a guide to understanding the system's architecture and key components.

docs/operator.md

@@ -0,0 +1,123 @@
+# Operator Module
+
+The `operator.py` module provides tools for managing and operating on datasets using an agent-based approach. It is designed to facilitate the execution of operations on datasets through a structured and validated process.
+
+## Classes
+
+### AgentSpecification
+
+Defines the specification for an LLM/agent:
+
+- `name`: Name of the LLM/agent
+- `version`: Version of the LLM/agent
+- `description`: Description of the LLM/agent
+- `capabilities`: List of capabilities
+- `configuration`: Configuration settings
+
+### OperatorToolBox
+
+Main class for dataset operations:
+
+- `__init__(spec: AgentSpecification, datasets: list[dict[str, Any]])`: Initialize with agent spec and datasets. This sets up the toolbox with the necessary specifications and datasets for operation.
+- `get_spec()`: Get the agent specification. Returns the `AgentSpecification` object associated with the toolbox.
+- `get_datasets()`: Get the datasets. Returns a list of datasets that the toolbox operates on.
+- `validate()`: Validate the toolbox. Checks if the toolbox is correctly set up with valid specifications and datasets.
+- `stop()`: Stop the toolbox. Halts any ongoing operations within the toolbox.
+- `run()`: Run the toolbox. Initiates the execution of operations as defined in the toolbox.
+- `get_results()`: Get operation results. Retrieves the results of operations performed by the toolbox.
+- `get_failures()`: Get failures. Provides a list of any failures encountered during operations.
+- `run_operation(operation: str)`: Run a specific operation. Executes a given operation on the datasets, returning the result or failure message.
+
+## Agent Tools
+
+The `dataset_manager_agent` provides these tools:
+
+### validate_toolbox
+
+Validates the OperatorToolBox:
+
+```python
+@dataset_manager_agent.tool
+async def validate_toolbox(ctx: RunContext[OperatorToolBox]) -> str
+```
+
+### execute_operation
+
+Executes an operation on a dataset:
+
+```python
+@dataset_manager_agent.tool
+async def execute_operation(ctx: RunContext[OperatorToolBox], operation: str) -> str
+```
+
+### retrieve_results
+
+Retrieves operation results:
+
+```python
+@dataset_manager_agent.tool
+async def retrieve_results(ctx: RunContext[OperatorToolBox]) -> str
+```
+
+### retrieve_failures
+
+Retrieves failures:
+
+```python
+@dataset_manager_agent.tool
+async def retrieve_failures(ctx: RunContext[OperatorToolBox]) -> str
+```
+
+## Usage Examples
+
+### Initializing the OperatorToolBox
+
+To initialize the `OperatorToolBox`, you need to provide an `AgentSpecification` and a list of datasets:
+
+```python
+spec = AgentSpecification(
+    name="GPT-4",
+    version="4.0",
+    description="A powerful language model",
+    capabilities=["text-generation", "question-answering"],
+    configuration={"max_tokens": 100},
+)
+
+datasets = [{"name": "dataset1"}, {"name": "dataset2"}]
+
+toolbox = OperatorToolBox(spec=spec, datasets=datasets)
+```
+
+### Synchronous Usage
+
+```python
+def run_dataset_manager_agent_sync():
+    prompts = [
+        "Validate the toolbox.",
+        "Execute operation on 'dataset2'.",
+        "Retrieve the results.",
+        "Retrieve any failures."
+    ]
+
+    for prompt in prompts:
+        result = dataset_manager_agent.run_sync(prompt, deps=toolbox)
+        print(f"Response: {result.data}")
+```
+
+### Asynchronous Usage
+
+```python
+async def run_dataset_manager_agent_async():
+    prompts = [
+        "Validate the toolbox.",
+        "Execute operation on 'dataset2'.",
+        "Retrieve the results.",
+        "Retrieve any failures."
+    ]
+
+    for prompt in prompts:
+        result = await dataset_manager_agent.run(prompt, deps=toolbox)
+        print(f"Response: {result.data}")
+```
+
+These updates provide a more detailed and comprehensive understanding of the `operator.py` module, its classes, and its usage.

docs/quickstart.md

@@ -0,0 +1,65 @@
+# Quickstart Guide
+
+Welcome to the Quickstart Guide for Agentic Security. This guide will help you set up and start using the project quickly.
+
+## Installation
+
+To get started with Agentic Security, install the package using pip:
+
+```shell
+pip install agentic_security
+```
+
+## Initial Setup
+
+After installation, you can start the application using the following command:
+
+```shell
+agentic_security
+```
+
+This will initialize the server and prepare it for use.
+
+## Basic Usage
+
+To run the main application, use:
+
+```shell
+python -m agentic_security
+```
+
+You can also view help options with:
+
+```shell
+agentic_security --help
+```
+
+## Running as a CI Check
+
+Initialize the configuration for CI checks:
+
+```shell
+agentic_security init
+```
+
+This will generate a default configuration file named `agesec.toml`.
+
+## Additional Commands
+
+- List available modules:
+
+  ```shell
+  agentic_security ls
+  ```
+
+- Run a security scan:
+
+  ```shell
+  agentic_security ci
+  ```
+
+## Further Information
+
+For more detailed information, refer to the [Documentation](index.md) or the [API Reference](api_reference.md).
+
+This quickstart guide should help you get up and running with Agentic Security efficiently.

mkdocs.yml

@@ -8,9 +8,13 @@ repo_name: msoedov/agentic_security
 copyright: Maintained by <a href="https://msoedov.github.io">Agentic Security Team</a>.
 
 nav:
-  - Home: index.md
+  - Adventure starts here:
+      - Overview: index.md
+      - Quickstart: quickstart.md
+      - Design: design.md
+      - Abstractions: abstractions.md
   - Features: probe_data.md
-  - Core Concepts:
+  - Concepts:
     - Probe Actor: probe_actor.md
     - Refusal Actor: refusal_classifier_plugins.md
     - Agent Spec: http_spec.md
@@ -26,10 +30,32 @@ nav:
     - Image Generation: image_generation.md
     - Stenography Functions: stenography.md
     - Reinforcement Learning Optimization: rl_model.md
+  - WIP:
+    - Agent Operator: operator.md
   - Reference:
     - API Reference: api_reference.md
-  - Community:
-    - Contributing: contributing.md
+  # - Project:
+  #     - Setup: setup.md
+  #     - Version control: version_control.md
+  #     - Docker: docker.md
+  #     - Variables: variables.md
+  #     - Custom libraries: custom_libraries.md
+  #     - Database: database.md
+  #     - Credentials: credentials.md
+  #     - Code execution: code_execution.md
+  #     - Settings: settings.md
+  #     - Version upgrades: version_upgrades.md
+  # - Contributing:
+  #     - Overview: contributing_overview.md
+  #     - Dev environment: dev_environment.md
+  #     - Backend: backend.md
+  #     - Frontend: frontend.md
+  #     - Documentation: documentation.md
+  # - About:
+  #     - Code of conduct: code_of_conduct.md
+  #     - Usage statistics: usage_statistics.md
+  #     - FAQ: faq.md
+  #     - Changelog: changelog.md
 
 plugins:
   - search

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
 
 [[package]]
 name = "aiohappyeyeballs"
@@ -645,21 +645,21 @@ tests = ["pytest", "pytest-cov", "pytest-xdist"]
 
 [[package]]
 name = "datasets"
-version = "3.2.0"
+version = "3.3.0"
 description = "HuggingFace community-driven open-source library of datasets"
 optional = false
 python-versions = ">=3.9.0"
 files = [
-    {file = "datasets-3.2.0-py3-none-any.whl", hash = "sha256:f3d2ba2698b7284a4518019658596a6a8bc79f31e51516524249d6c59cf0fe2a"},
-    {file = "datasets-3.2.0.tar.gz", hash = "sha256:9a6e1a356052866b5dbdd9c9eedb000bf3fc43d986e3584d9b028f4976937229"},
+    {file = "datasets-3.3.0-py3-none-any.whl", hash = "sha256:22312d09626f8fc3aa0a237b0c164997f5903bddd4c4c9e27dbaf563754c681b"},
+    {file = "datasets-3.3.0.tar.gz", hash = "sha256:54c607b06f6eaa1572e21e200d2870d89d50e3bcc622dc2021a53a6ce4f684c2"},
 ]
 
 [package.dependencies]
 aiohttp = "*"
 dill = ">=0.3.0,<0.3.9"
 filelock = "*"
-fsspec = {version = ">=2023.1.0,<=2024.9.0", extras = ["http"]}
-huggingface-hub = ">=0.23.0"
+fsspec = {version = ">=2023.1.0,<=2024.12.0", extras = ["http"]}
+huggingface-hub = ">=0.24.0"
 multiprocess = "<0.70.17"
 numpy = ">=1.17"
 packaging = "*"
@@ -673,15 +673,15 @@ xxhash = "*"
 [package.extras]
 audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0)"]
 benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"]
-dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"]
+dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "torchdata", "transformers", "transformers (>=4.42.0)", "zstandard"]
 docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"]
 jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"]
 quality = ["ruff (>=0.3.0)"]
 s3 = ["s3fs"]
 tensorflow = ["tensorflow (>=2.6.0)"]
 tensorflow-gpu = ["tensorflow (>=2.6.0)"]
-tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"]
-tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"]
+tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"]
+tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "decord (==0.6.0)", "elasticsearch (>=7.17.12,<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "torchdata", "transformers (>=4.42.0)", "zstandard"]
 torch = ["torch"]
 vision = ["Pillow (>=9.4.0)"]
 
@@ -784,13 +784,13 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth
 
 [[package]]
 name = "fastapi"
-version = "0.115.7"
+version = "0.115.8"
 description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "fastapi-0.115.7-py3-none-any.whl", hash = "sha256:eb6a8c8bf7f26009e8147111ff15b5177a0e19bb4a45bc3486ab14804539d21e"},
-    {file = "fastapi-0.115.7.tar.gz", hash = "sha256:0f106da6c01d88a6786b3248fb4d7a940d071f6f488488898ad5d354b25ed015"},
+    {file = "fastapi-0.115.8-py3-none-any.whl", hash = "sha256:753a96dd7e036b34eeef8babdfcfe3f28ff79648f86551eb36bfc1b0bf4a8cbf"},
+    {file = "fastapi-0.115.8.tar.gz", hash = "sha256:0ce9111231720190473e222cdf0f07f7206ad7e53ea02beb1d2dc36e2f0741e9"},
 ]
 
 [package.dependencies]
@@ -1055,6 +1055,25 @@ python-dateutil = ">=2.8.1"
 [package.extras]
 dev = ["flake8", "markdown", "twine", "wheel"]
 
+[[package]]
+name = "gtts"
+version = "2.5.4"
+description = "gTTS (Google Text-to-Speech), a Python library and CLI tool to interface with Google Translate text-to-speech API"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "gTTS-2.5.4-py3-none-any.whl", hash = "sha256:5dd579377f9f5546893bc26315ab1f846933dc27a054764b168f141065ca8436"},
+    {file = "gtts-2.5.4.tar.gz", hash = "sha256:f5737b585f6442f677dbe8773424fd50697c75bdf3e36443585e30a8d48c1884"},
+]
+
+[package.dependencies]
+click = ">=7.1,<8.2"
+requests = ">=2.27,<3"
+
+[package.extras]
+docs = ["sphinx", "sphinx-autobuild", "sphinx-click", "sphinx-mdinclude", "sphinx-rtd-theme"]
+tests = ["pytest (>=7.1.3,<8.4.0)", "pytest-cov", "testfixtures"]
+
 [[package]]
 name = "h11"
 version = "0.14.0"
@@ -1880,13 +1899,13 @@ pygments = ">2.12.0"
 
 [[package]]
 name = "mkdocs-material"
-version = "9.6.2"
+version = "9.6.4"
 description = "Documentation that simply works"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "mkdocs_material-9.6.2-py3-none-any.whl", hash = "sha256:71d90dbd63b393ad11a4d90151dfe3dcbfcd802c0f29ce80bebd9bbac6abc753"},
-    {file = "mkdocs_material-9.6.2.tar.gz", hash = "sha256:a3de1c5d4c745f10afa78b1a02f917b9dce0808fb206adc0f5bb48b58c1ca21f"},
+    {file = "mkdocs_material-9.6.4-py3-none-any.whl", hash = "sha256:414e8376551def6d644b8e6f77226022868532a792eb2c9accf52199009f568f"},
+    {file = "mkdocs_material-9.6.4.tar.gz", hash = "sha256:4d1d35e1c1d3e15294cb7fa5d02e0abaee70d408f75027dc7be6e30fb32e6867"},
 ]
 
 [package.dependencies]
@@ -1920,23 +1939,22 @@ files = [
 
 [[package]]
 name = "mkdocstrings"
-version = "0.27.0"
+version = "0.28.1"
 description = "Automatic documentation from sources, for MkDocs."
 optional = false
 python-versions = ">=3.9"
 files = [
-    {file = "mkdocstrings-0.27.0-py3-none-any.whl", hash = "sha256:6ceaa7ea830770959b55a16203ac63da24badd71325b96af950e59fd37366332"},
-    {file = "mkdocstrings-0.27.0.tar.gz", hash = "sha256:16adca6d6b0a1f9e0c07ff0b02ced8e16f228a9d65a37c063ec4c14d7b76a657"},
+    {file = "mkdocstrings-0.28.1-py3-none-any.whl", hash = "sha256:a5878ae5cd1e26f491ff084c1f9ab995687d52d39a5c558e9b7023d0e4e0b740"},
+    {file = "mkdocstrings-0.28.1.tar.gz", hash = "sha256:fb64576906771b7701e8e962fd90073650ff689e95eb86e86751a66d65ab4489"},
 ]
 
 [package.dependencies]
-click = ">=7.0"
 Jinja2 = ">=2.11.1"
 Markdown = ">=3.6"
 MarkupSafe = ">=1.1"
 mkdocs = ">=1.4"
-mkdocs-autorefs = ">=1.2"
-platformdirs = ">=2.2"
+mkdocs-autorefs = ">=1.3"
+mkdocs-get-deps = ">=0.2"
 pymdown-extensions = ">=6.3"
 
 [package.extras]
@@ -3772,6 +3790,62 @@ dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodest
 doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<=7.3.7)", "sphinx-design (>=0.4.0)"]
 test = ["Cython", "array-api-strict (>=2.0)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]
 
+[[package]]
+name = "sentry-sdk"
+version = "2.22.0"
+description = "Python client for Sentry (https://sentry.io)"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "sentry_sdk-2.22.0-py2.py3-none-any.whl", hash = "sha256:3d791d631a6c97aad4da7074081a57073126c69487560c6f8bffcf586461de66"},
+    {file = "sentry_sdk-2.22.0.tar.gz", hash = "sha256:b4bf43bb38f547c84b2eadcefbe389b36ef75f3f38253d7a74d6b928c07ae944"},
+]
+
+[package.dependencies]
+certifi = "*"
+urllib3 = ">=1.26.11"
+
+[package.extras]
+aiohttp = ["aiohttp (>=3.5)"]
+anthropic = ["anthropic (>=0.16)"]
+arq = ["arq (>=0.23)"]
+asyncpg = ["asyncpg (>=0.23)"]
+beam = ["apache-beam (>=2.12)"]
+bottle = ["bottle (>=0.12.13)"]
+celery = ["celery (>=3)"]
+celery-redbeat = ["celery-redbeat (>=2)"]
+chalice = ["chalice (>=1.16.0)"]
+clickhouse-driver = ["clickhouse-driver (>=0.2.0)"]
+django = ["django (>=1.8)"]
+falcon = ["falcon (>=1.4)"]
+fastapi = ["fastapi (>=0.79.0)"]
+flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"]
+grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"]
+http2 = ["httpcore[http2] (==1.*)"]
+httpx = ["httpx (>=0.16.0)"]
+huey = ["huey (>=2)"]
+huggingface-hub = ["huggingface_hub (>=0.22)"]
+langchain = ["langchain (>=0.0.210)"]
+launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"]
+litestar = ["litestar (>=2.0.0)"]
+loguru = ["loguru (>=0.5)"]
+openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"]
+openfeature = ["openfeature-sdk (>=0.7.1)"]
+opentelemetry = ["opentelemetry-distro (>=0.35b0)"]
+opentelemetry-experimental = ["opentelemetry-distro"]
+pure-eval = ["asttokens", "executing", "pure_eval"]
+pymongo = ["pymongo (>=3.1)"]
+pyspark = ["pyspark (>=2.4.4)"]
+quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]
+rq = ["rq (>=0.6)"]
+sanic = ["sanic (>=0.8)"]
+sqlalchemy = ["sqlalchemy (>=1.2)"]
+starlette = ["starlette (>=0.19.1)"]
+starlite = ["starlite (>=1.48)"]
+statsig = ["statsig (>=0.55.3)"]
+tornado = ["tornado (>=6)"]
+unleash = ["UnleashClient (>=6.0.1)"]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -4365,4 +4439,4 @@ propcache = ">=0.2.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "211d8b41dfd43afee62345619497bd7b6b64dad2b39ad2013c47beafd3f0a26b"
+content-hash = "a741ff960d86175204b90cdb4f935d3873a6a38d2d547c1ded73c17ab54b4312"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agentic_security"
-version = "0.4.4"
+version = "0.5.0"
 description = "Agentic LLM vulnerability scanner"
 authors = ["Alexander Miasoiedov <msoedov@gmail.com>"]
 maintainers = ["Alexander Miasoiedov <msoedov@gmail.com>"]
@@ -28,14 +28,14 @@ agentic_security = "agentic_security.__main__:main"
 
 [tool.poetry.dependencies]
 python = "^3.11"
-fastapi = "^0.115.6"
+fastapi = "^0.115.8"
 uvicorn = "^0.34.0"
 fire = "0.7.0"
 loguru = "^0.7.3"
 httpx = "^0.28.1"
 cache-to-disk = "^2.0.0"
 pandas = ">=1.4,<3.0"
-datasets = ">=1.14,<4.0"
+datasets = "^3.3.0"
 tabulate = ">=0.8.9,<0.10.0"
 colorama = "^0.4.4"
 matplotlib = "^3.9.2"
@@ -47,6 +47,8 @@ jinja2 = "^3.1.4"
 python-multipart = "^0.0.20"
 tomli = "^2.2.1"
 rich = "13.9.4"
+gTTS = "^2.5.4"
+sentry_sdk = "^2.22.0"
 # garak = { version = "*", optional = true }
 
 
@@ -66,7 +68,7 @@ huggingface-hub = ">=0.25.1,<0.29.0"
 
 # Docs
 mkdocs = ">=1.4.2"
-mkdocs-material = ">=8.5.10"
+mkdocs-material = "^9.6.4"
 mkdocstrings = ">=0.26.1"
 mkdocs-jupyter = ">=0.25.1"
 

ui/.env.example

@@ -0,0 +1 @@
+VUE_APP_SERVER_URL=''#replace this with  url at which agentic_security server is running 

ui/.eslintrc.js

@@ -0,0 +1,25 @@
+module.exports = {
+    env: {
+      browser: true,
+      es2021: true,
+      node :true
+    },
+    extends: [
+      'eslint:recommended',
+      'plugin:vue/essential',
+    ],
+    parserOptions: {
+      ecmaVersion: 12,
+      sourceType: 'module',
+    },
+    plugins: [
+      'vue',
+    ],
+    rules: {
+      'no-unused-vars': 'off', // Disable the rule
+      'no-constant-condition': 'off',
+      'no-global-assign': 'off',
+      // or
+      // 'no-unused-vars': 'warn', // Change the rule to a warning
+    },
+  };

ui/.gitignore

@@ -0,0 +1,23 @@
+.DS_Store
+node_modules
+/dist
+
+
+# local env files
+.env.local
+.env.*.local
+
+# Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

ui/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

ui/jsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "module": "esnext",
+    "baseUrl": "./",
+    "moduleResolution": "node",
+    "paths": {
+      "@/*": [
+        "src/*"
+      ]
+    },
+    "lib": [
+      "esnext",
+      "dom",
+      "dom.iterable",
+      "scripthost"
+    ]
+  }
+}

ui/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "agentic-vulnerability-scanner-llm-ui",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve ",
+    "dev": "vue-cli-service serve ",
+    "build": "vue-cli-service build",
+    "lint": "vue-cli-service lint"
+  },
+  "dependencies": {
+    "core-js": "^3.8.3",
+    "lucide": "^0.474.0",
+    "vue": "^3.2.13"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.12.16",
+    "@babel/eslint-parser": "^7.12.16",
+    "@vue/cli-plugin-babel": "~5.0.0",
+    "@vue/cli-plugin-eslint": "~5.0.0",
+    "@vue/cli-service": "~5.0.0",
+    "eslint": "^7.32.0",
+    "eslint-plugin-vue": "^8.0.3"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/vue3-essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "@babel/eslint-parser"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead",
+    "not ie 11"
+  ]
+}

ui/public/base.js

@@ -0,0 +1,232 @@
+
+let URL = window.location.href;
+if (URL.endsWith('/')) {
+  URL = URL.slice(0, -1);
+}
+URL = process.env.VUE_APP_SERVER_URL
+
+// Vue application
+ let LLM_SPECS = [
+  `POST ${URL}/v1/self-probe
+Authorization: Bearer XXXXX
+Content-Type: application/json
+
+{
+"prompt": "<<PROMPT>>"
+}
+
+`,
+  `POST https://api.openai.com/v1/chat/completions
+Authorization: Bearer $OPENAI_API_KEY
+Content-Type: application/json
+
+{
+"model": "gpt-3.5-turbo",
+"messages": [{"role": "user", "content": "<<PROMPT>>"}],
+"temperature": 0.7
+}
+`,
+  `
+POST https://api.deepseek.com/chat/completions
+Authorization: Bearer $DEEPSEEK_API_KEY
+Content-Type: application/json
+
+{
+  "model": "deepseek-chat",
+  "messages": [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": "<<PROMPT>>"}
+  ],
+  "stream": false
+}
+`,
+  `POST https://api.replicate.com/v1/models/mistralai/mixtral-8x7b-instruct-v0.1/predictions
+Authorization: Bearer $APIKEY
+Content-Type: application/json
+
+{
+"input": {
+"top_k": 50,
+"top_p": 0.9,
+"prompt": "Write a bedtime story about neural networks I can read to my toddler",
+"temperature": 0.6,
+"max_new_tokens": 1024,
+"prompt_template": "<s>[INST] <<PROMPT>> [/INST] ",
+"presence_penalty": 0,
+"frequency_penalty": 0
+}
+}
+`,
+  `POST https://api.groq.com/v1/request_manager/text_completion
+Authorization: Bearer  XXXXX
+Content-Type: application/json
+
+{
+"model_id": "codellama-34b",
+"system_prompt": "You are helpful and concise coding assistant",
+"user_prompt": "<<PROMPT>>"
+}
+`,
+  `POST https://api.together.xyz/v1/chat/completions
+Authorization: Bearer $TOGETHER_API_KEY
+Content-Type: application/json
+
+{
+"model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
+"messages": [
+{"role": "system", "content": "You are an expert travel guide"},
+{"role": "user", "content": "<<PROMPT>>"}
+]
+}
+`,
+  `POST ${URL}/v1/self-probe-image
+Authorization: Bearer XXXXX
+Content-Type: application/json
+
+[
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": "What is in this image?",
+        },
+        {
+          "type": "image_url",
+          "image_url": {
+            "url":  f"data:image/jpeg;base64,{<<BASE64_IMAGE>>}"
+          },
+        },
+      ],
+    }
+]
+`,
+  `POST ${URL}/v1/self-probe-file
+Authorization: Bearer $GROQ_API_KEY
+Content-Type: multipart/form-data
+
+{
+  "file": "@./sample_audio.m4a",
+  "model": "whisper-large-v3"
+}
+`,
+  `POST https://api.gemini.com/v1/generate
+Authorization: Bearer $GEMINI_API_KEY
+Content-Type: application/json
+
+{
+  "model": "gemini-latest",
+  "prompt": "<<PROMPT>>",
+  "temperature": 0.8,
+  "max_tokens": 150,
+  "top_p": 1.0,
+  "frequency_penalty": 0,
+  "presence_penalty": 0
+}
+`,
+  `POST https://api.anthropic.com/v1/complete
+Authorization: Bearer $ANTHROPIC_API_KEY
+Content-Type: application/json
+
+{
+  "model": "claude-v1.3",
+  "prompt": "<<PROMPT>>",
+  "temperature": 0.7,
+  "max_tokens_to_sample": 256,
+  "stop_sequences": ["\n\nHuman:"]
+}
+`,
+  `POST https://api.cohere.ai/generate
+Authorization: Bearer $COHERE_API_KEY
+Content-Type: application/json
+
+{
+  "model": "command-xlarge-nightly",
+  "prompt": "<<PROMPT>>",
+  "max_tokens": 300,
+  "temperature": 0.75,
+  "k": 0,
+  "p": 0.75
+}
+`,
+
+  `POST https://<<RESOURCE_NAME>>.openai.azure.com/openai/deployments/<<DEPLOYMENT_NAME>>/completions?api-version=2023-06-01-preview
+Authorization: Bearer $AZURE_API_KEY
+Content-Type: application/json
+
+{
+  "prompt": "<<PROMPT>>",
+  "max_tokens": 150,
+  "temperature": 0.7,
+  "top_p": 0.9,
+  "frequency_penalty": 0,
+  "presence_penalty": 0
+}
+`,
+
+  `POST https://api.assemblyai.com/v2/transcript
+Authorization: Bearer $ASSEMBLY_API_KEY
+Content-Type: application/json
+
+{
+  "audio_url": "<<AUDIO_FILE_URL>>"
+}
+`,
+
+]
+
+ let LLM_CONFIGS = [
+  { name: 'Custom API', prompts: 40000, customInstructions: 'Requires api spec' },
+  { name: 'Open AI', prompts: 24000 },
+  { name: 'Deepseek v1', prompts: 24000 },
+  { name: 'Replicate', prompts: 40000 },
+  { name: 'Groq', prompts: 40000 },
+  { name: 'Together.ai', prompts: 40000 },
+  { name: 'Custom API Image', prompts: 40000, customInstructions: 'Requires api spec', modality: 'Image' },
+  { name: 'Custom API Files', prompts: 40000, customInstructions: 'Requires api spec', modality: 'Files' },
+  { name: 'Gemini', prompts: 40000 },
+  { name: 'Claude', prompts: 40000 },
+  { name: 'Cohere', prompts: 40000 },
+  { name: 'Azure OpenAI', prompts: 40000 },
+  { name: 'assemblyai', prompts: 40000 },
+
+
+]
+
+function has_image(spec) {
+  return spec.includes('<<BASE64_IMAGE>>');
+}
+
+function has_files(spec) {
+  return spec.includes('multipart/form-data');
+}
+
+
+function _getFailureRateColor(failureRate) {
+  // We're now working with the strength percentage, so no need to invert
+  const strengthRate = 100 - failureRate;
+
+  if (strengthRate >= 95) return 'text-green-400';
+  else if (strengthRate >= 85) return 'text-green-400';
+  else if (strengthRate >= 75) return 'text-green-500';
+  else if (strengthRate >= 65) return 'text-yellow-400';
+  else if (strengthRate >= 55) return 'text-yellow-500';
+  else if (strengthRate >= 45) return 'text-orange-400';
+  else if (strengthRate >= 35) return 'text-orange-500';
+  else if (strengthRate >= 25) return 'text-dark-accent-red';
+  else if (strengthRate >= 15) return 'text-red-400';
+  else if (strengthRate > 0) return 'text-red-500';
+  else return 'text-gray-100'; // This can be the default for strengthRate of 0 or less
+}
+
+function _getFailureRateScore(failureRate) {
+  // Convert failureRate to a strength percentage
+  const strengthRate = 100 - failureRate;
+
+  if (strengthRate >= 90) return 'A';
+  else if (strengthRate >= 80) return 'B';
+  else if (strengthRate >= 70) return 'C';
+  else if (strengthRate >= 60) return 'D';
+  else return 'E'; // For strengthRate less than 60
+}
+export { LLM_SPECS, LLM_CONFIGS, has_image, has_files, _getFailureRateColor, _getFailureRateScore ,URL };

ui/public/index.html

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en" class="dark">
+  <header>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>LLM Vulnerability Scanner</title>
+   
+    <script src="https://unpkg.com/lucide@latest/dist/umd/lucide.js"></script>
+    <link href="https://fonts.cdnfonts.com/css/technopollas" rel="stylesheet">
+    <link href="styles/output.css" rel="stylesheet">
+
+  </header>
+  <body class="bg-dark-bg text-dark-text font-sans"> 
+   
+    <noscript>
+      <strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
+    </noscript>
+    <div id="vue-app" class="min-h-screen p-8"></div>
+  
+   
+  </body>
+</html>

ui/public/styles/styles.css

@@ -0,0 +1,11 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&display=swap');
+.scrollbar-hide::-webkit-scrollbar {
+  display: none;
+}
+.scrollbar-hide {
+  -ms-overflow-style: none;  /* IE and Edge */
+  scrollbar-width: none;     /* Firefox */
+}

ui/public/telemetry.js

@@ -0,0 +1,4 @@
+!function (t, e) { var o, n, p, r; e.__SV || (window.posthog = e, e._i = [], e.init = function (i, s, a) { function g(t, e) { var o = e.split("."); 2 == o.length && (t = t[o[0]], e = o[1]), t[e] = function () { t.push([e].concat(Array.prototype.slice.call(arguments, 0))) } } (p = t.createElement("script")).type = "text/javascript", p.async = !0, p.src = s.api_host.replace(".i.posthog.com", "-assets.i.posthog.com") + "/static/array.js", (r = t.getElementsByTagName("script")[0]).parentNode.insertBefore(p, r); var u = e; for (void 0 !== a ? u = e[a] = [] : a = "posthog", u.people = u.people || [], u.toString = function (t) { var e = "posthog"; return "posthog" !== a && (e += "." + a), t || (e += " (stub)"), e }, u.people.toString = function () { return u.toString(1) + ".people (stub)" }, o = "init push capture register register_once register_for_session unregister unregister_for_session getFeatureFlag getFeatureFlagPayload isFeatureEnabled reloadFeatureFlags updateEarlyAccessFeatureEnrollment getEarlyAccessFeatures on onFeatureFlags onSessionId getSurveys getActiveMatchingSurveys renderSurvey canRenderSurvey getNextSurveyStep identify setPersonProperties group resetGroups setPersonPropertiesForFlags resetPersonPropertiesForFlags setGroupPropertiesForFlags resetGroupPropertiesForFlags reset get_distinct_id getGroups get_session_id get_session_replay_url alias set_config startSessionRecording stopSessionRecording sessionRecordingStarted loadToolbar get_property getSessionProperty createPersonProfile opt_in_capturing opt_out_capturing has_opted_in_capturing has_opted_out_capturing clear_opt_in_out_capturing debug".split(" "), n = 0; n < o.length; n++)g(u, o[n]); e._i.push([i, s, a]) }, e.__SV = 1) }(document, window.posthog || []);
+window.posthog.init('phc_jfYo5xEofW7eJtiU8rLt2Z8jw1E2eW27BxwTJzwRufH', {
+    api_host: 'https://us.i.posthog.com', person_profiles: 'identified_only' // or 'always' to create profiles for anonymous users as well
+})

ui/src/App.vue

@@ -0,0 +1,52 @@
+<template>
+  <div>
+         <div
+        class="bg-dark-accent-green text-dark-bg py-4 px-6 rounded-lg mb-28 text-center">
+        <h4 class="text-lg font-semibold">
+          🚀 NEW: Star Agentic Security on
+          <a href="https://github.com/msoedov/agentic_security" target="_blank"
+            class="underline" data-faitracker-click-bind="true">Github</a> 🚀
+        </h4>
+      </div>
+
+      <!-- Header with Github link -->
+      <header class="flex justify-between items-center mb-8 relative"
+        v-if="false">
+        <div class="w-full absolute left-0 flex justify-center">
+          <h1
+            class="text-2xl font-bold text-gray-400"> <span
+              class="text-2xl font-technopollas  text-gray-300">Agentic
+            </span>
+
+            Vulnerability
+            Scanner</h1>
+        </div>
+</header>
+<PageContent/>
+<PageConfigs/>
+    <PageFooter />
+    </div>
+  
+</template>
+
+<script>
+import PageFooter from "./components/PageFooter.vue";
+import PageContent from "./components/PageContent.vue";
+import PageConfigs from "./components/PageConfigs.vue";
+
+
+
+
+
+export default {
+  components: {
+    PageFooter,
+    PageContent,
+    PageConfigs
+  }
+};
+</script>
+
+<style scoped>
+/* Global styles or App.vue specific styles */
+</style>

ui/src/components/LLMSpecInput.vue

@@ -0,0 +1,58 @@
+<template>
+  <section class="bg-dark-card rounded-lg p-6 shadow-lg">
+    <div @click="toggleLLMSpec" class="flex justify-between items-center cursor-pointer">
+      <h2 class="text-2xl font-bold">LLM API Spec</h2>
+      </div>
+
+    <div v-show="showLLMSpec" class="mt-4">
+      <label v-if="isFocused" for="llm-spec" class="block text-sm font-medium mb-2">
+        LLM API Spec, PROMPT variable will be replaced with the testing prompt
+      </label>
+      </div>
+  </section>
+</template>
+
+<script>
+export default {
+  name: 'LLMSpecInput',
+  data() {
+    return {
+      showLLMSpec: false,
+      isFocused: false,
+       modelSpec: '',
+      errorMsg: null,
+      okMsg: null,
+    };
+  },
+  methods: {
+       toggleLLMSpec() {
+      this.showLLMSpec = !this.showLLMSpec;
+    },
+    focusTextarea() {
+      this.isFocused = true;
+    },
+    unfocusTextarea() {
+      this.isFocused = false;
+    },
+    adjustHeight(event) {
+      event.target.style.height = 'auto';
+      event.target.style.height = event.target.scrollHeight + 'px';
+    },
+    verifyIntegration() {
+      // Your logic for verifying integration
+    },
+  },
+  computed: {
+    highlightedText() {
+      // Your logic for highlighted text
+    },
+     statusDotClass() {
+      // Your logic for status dot class
+    },
+  },
+};
+</script>
+
+<style scoped>
+/* Styles for the LLM Spec Input */
+</style>

ui/src/components/PageConfigs.vue

@@ -0,0 +1,907 @@
+<template>
+     <main class="max-w-6xl mx-auto space-y-8">
+        <section class="bg-dark-card rounded-lg p-6 shadow-lg">
+          <h2 class="text-2xl font-bold mb-4">Select a Config</h2>
+
+          <div class="flex space-x-4 overflow-x-auto scrollbar-hide">
+            <div
+              v-for="(config, index) in configs"
+              :key="index"
+              @click="selectConfig(index)"
+              class="flex-none w-1/2 sm:w-1/3 md:w-1/4 lg:w-1/5 border-2 rounded-lg p-4 flex flex-col items-start transition-all hover:shadow-md cursor-pointer"
+              :class="{
+        'border-dark-accent-green': selectedConfig === index,
+        'border-gray-600': selectedConfig !== index
+      }">
+              <div class="font-medium mb-2">{{ config.name }}</div>
+              <div class="text-sm text-gray-400">
+                {{ config.customInstructions || 'Requires API key' }}
+              </div>
+              <div class="mt-2 text-dark-accent-green font-semibold">
+                {{config.modality || 'API'}}</div>
+            </div>
+          </div>
+        </section>
+
+        <!-- Collapsible LLM Spec Input -->
+        <section class="bg-dark-card rounded-lg p-6 shadow-lg" >
+          <div @click="toggleLLMSpec"
+            class="flex justify-between items-center cursor-pointer">
+
+            <h2 class="text-2xl font-bold">LLM API Spec</h2>
+            <span :class="statusDotClass"
+              class="w-3 h-3 rounded-full mr-2"></span>
+            <svg :class="{'rotate-180': showLLMSpec}"
+              class="w-6 h-6 transition-transform duration-200"
+              xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
+              stroke="currentColor" stroke-width="2" stroke-linecap="round"
+              stroke-linejoin="round">
+              <polyline points="6 9 12 15 18 9"></polyline>
+            </svg>
+          </div>
+
+          <div v-show="showLLMSpec" class="mt-4">
+            <label v-if="isFocused" for="llm-spec"
+              class="block text-sm font-medium mb-2">
+              LLM API Spec, PROMPT variable will be replaced with the testing
+              prompt
+            </label>
+            <div
+              v-if="!isFocused"
+              
+              class="w-full bg-dark-bg text-dark-accent-orange border border-gray-600 rounded-lg p-3 cursor-text mb-5"
+              @click="focusTextarea"
+              v-html="highlightedText"></div>
+
+            <textarea
+              v-else
+              ref="textarea"
+              class="w-full bg-dark-bg text-dark-accent-orange border border-gray-600 rounded-lg p-3 focus:outline-none focus:ring-2 focus:ring-dark-accent-green"
+              @blur="unfocusTextarea"
+              v-model="modelSpec"
+              @input="adjustHeight"
+              rows="5"
+              placeholder="Enter LLM API Spec here..."></textarea>
+
+            <!-- Error and Success Messages -->
+            <div v-if="errorMsg"
+              class="bg-dark-accent-red bg-opacity-20 border border-dark-accent-red text-dark-accent-red px-4 py-3 rounded-lg relative"
+              role="alert">
+              <strong class="font-bold">Oops!</strong>
+              <span class="block sm:inline">{{errorMsg}}</span>
+            </div>
+            <div v-if="okMsg"
+              class="bg-dark-accent-green bg-opacity-20 border border-dark-accent-green text-dark-accent-green px-4 py-3 rounded-lg relative"
+              role="alert">
+              <strong class="font-bold"></strong>
+              <span class="block sm:inline">{{okMsg}}</span>
+            </div>
+
+            <!-- Action Buttons -->
+            <section class="flex justify-center space-x-4 mt-10">
+              <button
+                @click="verifyIntegration"
+                class="bg-dark-accent-orange text-dark-bg rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors">
+                Verify Integration
+              </button>
+            </section>
+          </div>
+        </section>
+        <!-- LLM Spec Input -->
+     <section class="bg-dark-card rounded-lg p-6 shadow-lg" v-if="false" >
+          <h2 class="text-2xl font-bold mb-4">LLM API Spec</h2>
+          <label for="llm-spec" class="block text-sm font-medium mb-2">
+            LLM API Spec, PROMPT variable will be replaced with the testing
+            prompt
+          </label>
+          <textarea
+            class="w-full bg-dark-bg text-dark-accent-orange border border-gray-600 rounded-lg p-3 focus:outline-none focus:ring-2 focus:ring-dark-accent-green"
+            id="llm-spec"
+            ref="textarea"
+            v-model="modelSpec"
+            @input="adjustHeight"
+            rows="5"
+            placeholder="Enter LLM API Spec here..."></textarea>
+        </section>
+        <section
+          class="bg-dark-card rounded-lg p-6 shadow-lg mt-8 border-dark-accent-green border-2">
+          <div @click="toggleParams"
+            class="flex justify-between items-center cursor-pointer">
+            <div class="flex items-center">
+              <svg xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 mr-2"
+                fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                <path stroke-linecap="round" stroke-linejoin="round"
+                  stroke-width="2"
+                  d="M12 6V4m0 2a2 2 0 100 4m0-4a2 2 0 110 4m-6 8a2 2 0 100-4m0 4a2 2 0 110-4m0 4v2m0-6V4m6 6v10m6-2a2 2 0 100-4m0 4a2 2 0 110-4m0 4v2m0-6V4" />
+              </svg>
+              <h2 class="text-2xl font-bold">Parameters</h2>
+            </div>
+            <svg :class="{'rotate-180': showParams}"
+              class="w-6 h-6 transition-transform duration-200"
+              xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
+              stroke="currentColor" stroke-width="2" stroke-linecap="round"
+              stroke-linejoin="round">
+              <polyline points="6 9 12 15 18 9"></polyline>
+            </svg>
+          </div>
+          <div v-show="showParams" class="mt-4">
+            <div class="flex items-center justify-end mt-4">
+              <button
+                @click="confirmResetState"
+                class="flex items-center bg-dark-accent-red text-dark-bg rounded-lg px-4 py-2 text-sm font-medium hover:bg-opacity-80 transition-colors">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 mr-2"
+                  fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                  <path stroke-linecap="round" stroke-linejoin="round"
+                    stroke-width="2"
+                    d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
+                </svg>
+                Reset State
+              </button>
+            </div>
+            <!-- Confirmation Modal -->
+            <div
+              v-if="showResetConfirmation"
+              class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50">
+              <div class="bg-dark-card rounded-lg p-6 max-w-sm w-full">
+                <h3 class="text-xl font-bold mb-4 text-dark-text">Confirm
+                  Reset</h3>
+                <p class="text-gray-400 mb-6">Are you sure you want to reset all
+                  settings to their default state? This action cannot be
+                  undone.</p>
+                <div class="flex justify-end space-x-4">
+                  <button
+                    @click="showResetConfirmation = false"
+                    class="bg-gray-600 text-dark-text rounded-lg px-4 py-2 hover:bg-opacity-80 transition-colors">
+                    Cancel
+                  </button>
+                  <button
+                    @click="resetState"
+                    class="bg-dark-accent-red text-dark-bg rounded-lg px-4 py-2 hover:bg-opacity-80 transition-colors">
+                    Reset
+                  </button>
+                </div>
+              </div>
+            </div>
+            <!-- Confirmation Modal -->
+
+            <!-- Maximum Budget Slider -->
+            <!-- Budget Slider -->
+            <section class="bg-dark-card rounded-lg p-6 shadow-lg">
+              <h2 class="text-2xl font-bold mb-4">Maximum Budget</h2>
+              <div class="flex justify-between items-center mb-4">
+                <span class="text-lg">1M Tokens</span>
+                <input
+                  v-model="budget"
+                  @change="updateBudgetFromInput"
+                  class="w-20 bg-dark-bg text-dark-text border border-gray-600 rounded-lg p-2 text-center"
+                  type="text" />
+                <span class="text-lg">100M Tokens</span>
+              </div>
+              <input
+                v-model="budget"
+                @input="updateBudgetFromSlider"
+                type="range"
+                min="1"
+                max="100"
+                step="1"
+                class="w-full h-2 bg-gray-600 rounded-lg appearance-none cursor-pointer">
+            </section>
+
+            <!-- Optimize Toggle -->
+            <div class="flex flex-col mt-6 mr-10 ml-10">
+              <div class="flex items-center justify-between mb-2">
+                <h3 class="text-lg font-semibold">Optimize Test</h3>
+                <label class="relative inline-flex items-center cursor-pointer">
+                  <input type="checkbox" v-model="optimize"
+                    class="sr-only peer">
+                  <div
+                    class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-dark-accent-green rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-dark-accent-green"></div>
+                </label>
+              </div>
+              <p class="text-sm text-gray-400 mt-2 mb-6">
+                When enabled, this option runs a Bayesian optimization loop to
+                find the most effective test parameters. This can potentially
+                reduce the cost and the total running time of your vulnerability
+                scan, but may reduce accuracy.
+              </p>
+
+              <!-- Chart Diagram Toggle -->
+              <div class="flex items-center justify-between mb-2">
+                <h3 class="text-lg font-semibold">Enable Chart Diagram</h3>
+                <label class="relative inline-flex items-center cursor-pointer">
+                  <input type="checkbox" v-model="enableChartDiagram"
+                    class="sr-only peer">
+                  <div
+                    class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-dark-accent-green rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-dark-accent-green"></div>
+                </label>
+              </div>
+              <p class="text-sm text-gray-400 mt-2 mb-6">
+                When enabled, a chart diagram will be generated to visualize the
+                results of your vulnerability scan.
+              </p>
+
+              <!-- Logging Toggle -->
+              <div class="flex items-center justify-between mb-2">
+                <h3 class="text-lg font-semibold">Enable Detailed Logging</h3>
+                <label class="relative inline-flex items-center cursor-pointer">
+                  <input type="checkbox" v-model="enableLogging"
+                    class="sr-only peer">
+                  <div
+                    class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-dark-accent-green rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-dark-accent-green"></div>
+                </label>
+              </div>
+              <p class="text-sm text-gray-400 mt-2 mb-6">
+                When enabled, detailed logs will be generated during the
+                vulnerability scan process. This can be useful for debugging and
+                in-depth analysis.
+              </p>
+
+              <!-- Concurrency Toggle -->
+              <div class="flex items-center justify-between mb-2">
+                <h3 class="text-lg font-semibold">Enable Concurrency</h3>
+                <label class="relative inline-flex items-center cursor-pointer">
+                  <input type="checkbox" v-model="enableConcurrency"
+                    class="sr-only peer">
+                  <div
+                    class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-dark-accent-green rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-dark-accent-green"></div>
+                </label>
+              </div>
+              <p class="text-sm text-gray-400 mt-2">
+                When enabled, the vulnerability scan will run multiple tests
+                concurrently. This can significantly reduce the total scan time
+                but may increase resource usage.
+              </p>
+            </div>
+          </div>
+        </section>
+
+        <!-- Modules Selection -->
+        <section
+          class="bg-dark-card rounded-lg p-6 shadow-lg border-dark-accent-red border-4">
+          <div @click="toggleModules"
+            class="flex justify-between items-center cursor-pointer">
+            <h2 class="text-2xl font-bold">Modules [{{selectedDS}}
+              selected]</h2>
+            <svg :class="{'rotate-180': showModules}"
+              class="w-6 h-6 transition-transform duration-200"
+              xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
+              stroke="currentColor" stroke-width="2" stroke-linecap="round"
+              stroke-linejoin="round">
+              <polyline points="6 9 12 15 18 9"></polyline>
+            </svg>
+          </div>
+
+          <div v-show="showModules" class="mt-4">
+            <!-- Many-shot jailbreaking Toggle -->
+            <div v-if="enableMultiStepAttack" class="alert-box mt-4">
+              <div
+                class="bg-yellow-100 border border-yellow-400 text-yellow-700 px-4 py-3 rounded relative"
+                role="alert">
+                <strong class="font-bold">Notice:</strong>
+                <span class="block sm:inline">A many-shot attack might take a
+                  longer time to complete.
+                </span>
+              </div>
+            </div>
+            <div class="flex items-center justify-between mb-2 mt-10">
+              <h3 class="text-lg font-semibold">Enable Many-shot
+                jailbreaking</h3>
+
+              <label class="relative inline-flex items-center cursor-pointer">
+                <input type="checkbox" v-model="enableMultiStepAttack"
+                  class="sr-only peer">
+                <div
+                  class="w-11 h-6 bg-gray-200 peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-dark-accent-green rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-dark-accent-green"></div>
+              </label>
+            </div>
+            <p class="text-sm text-gray-400 mt-2 mb-2">
+              When enabled, the scan will attempt Many-shot jailbreaking
+              simulations
+            </p>
+
+            <div v-if="hasFileSpec" class="alert-box mt-10">
+              <div
+                class="bg-yellow-100 border border-yellow-400 text-yellow-700 px-4 py-3 rounded relative"
+                role="alert">
+                <strong class="font-bold">Notice:</strong>
+                <span class="block sm:inline">Converting audio or image prompts
+                  might
+                  take some time to compute.</span>
+              </div>
+            </div>
+
+            <div class="flex justify-between mb-4 mt-4">
+              <button @click="selectAllPackages"
+                class="text-dark-accent-green hover:underline">Select
+                All</button>
+              <button @click="deselectAllPackages"
+                class="text-gray-400 hover:underline">Deselect All</button>
+            </div>
+
+            <div class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4">
+              <div
+                v-for="(pkg, index) in dataConfig"
+                :key="index"
+                @click="addPackage(index)"
+                class="border rounded-lg p-3 cursor-pointer transition-all hover:shadow-md overflow-hidden"
+                :class="{
+                'border-dark-accent-green bg-dark-accent-green bg-opacity-20': pkg.selected,
+                'border-gray-600': !pkg.selected
+              }">
+                <div class="font-medium mb-1 truncate">{{ pkg.dataset_name
+                  }}</div>
+                <div class="text-sm text-gray-400 truncate">
+                  {{ pkg.source || 'Local dataset' }}
+                </div>
+                <div class="mt-2 text-sm font-semibold">
+                  {{ pkg.dynamic ? 'Dynamic dataset' :
+                  `${pkg.num_prompts.toLocaleString()} prompts` }}
+                </div>
+              </div>
+            </div>
+          </div>
+        </section>
+
+        <!-- Error and Success Messages -->
+        <div v-if="errorMsg"
+          class="bg-dark-accent-red bg-opacity-20 border border-dark-accent-red text-dark-accent-red px-4 py-3 rounded-lg relative"
+          role="alert">
+          <strong class="font-bold">Oops!</strong>
+          <span class="block sm:inline">{{errorMsg}}</span>
+        </div>
+        <div v-if="okMsg"
+          class="bg-dark-accent-green bg-opacity-20 border border-dark-accent-green text-dark-accent-green px-4 py-3 rounded-lg relative"
+          role="alert">
+          <strong class="font-bold">></strong>
+          <span class="block sm:inline">{{okMsg}}</span>
+        </div>
+
+        <!-- Action Buttons -->
+        <section class="flex justify-center space-x-4">
+          <button
+            @click="verifyIntegration"
+            class="bg-dark-accent-orange text-dark-bg rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors">
+            Verify Integration
+          </button>
+          <button
+            @click="startScan"
+            v-if="!scanRunning"
+            class="bg-dark-accent-green text-dark-bg rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors flex items-center">
+            <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24"
+              viewBox="0 0 24 24" fill="none" stroke="currentColor"
+              stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
+              class="mr-2"><polygon points="5 3 19 12 5 21 5 3"></polygon></svg>
+            Run Scan
+          </button>
+          <button
+            @click="stopScan"
+            v-if="scanRunning"
+            class="bg-dark-accent-red text-dark-bg rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors flex items-center">
+            <!-- Stop Icon -->
+            <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24"
+              viewBox="0 0 24 24" fill="none" stroke="currentColor"
+              stroke-width="2" stroke-linecap="round" stroke-linejoin="round"
+              class="mr-2"><rect x="6" y="6" width="12"
+                height="12"></rect></svg>
+            Stop Scan
+          </button>
+        </section>
+
+        <!-- Progress Bar -->
+        <div id="progress"
+          class="bg-dark-accent-green rounded-full h-2 transition-all duration-500 ease-in-out"
+          v-bind:style="{width: progressWidth}">
+        </div>
+
+        <!-- Scan Results -->
+        <section class="bg-dark-card rounded-lg p-6 shadow-lg"
+          v-if="mainTable.length > 0">
+          <h2 class="text-2xl font-bold mb-4">Scan Results</h2>
+          <div class="overflow-x-auto">
+            <table class="w-full text-left">
+              <thead>
+                <tr class="border-b border-gray-600">
+                  <th class="p-3">Vulnerability Module</th>
+                  <th class="p-3">% Strength</th>
+                  <th class="p-3">Number of Tokens</th>
+                  <th class="p-3">Cost (in gpt-3 tokens)</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr v-for="result in mainTable" :key="result.module || index" class="border-b border-gray-700"
+                  :class="{'text-dark-accent-green': result.last, 'text-gray-300': !result.last}">
+                  <td class="p-3">{{result.module}}</td>
+                  <td class="p-3 text-gray-100"
+                    :class="getFailureRateColor(result.failureRate)">
+                    {{getFailureRateScore(result.failureRate)}}( {{(100 -
+                    result.failureRate).toFixed(2)}} )
+                  </td>
+                  <td class="p-3">{{result.tokens}}k</td>
+                  <td class="p-3">${{result.cost.toFixed(2)}}</td>
+                </tr>
+              </tbody>
+            </table>
+          </div>
+        </section>
+
+        <!-- Download Button -->
+        <button
+          @click="downloadFailures"
+          class="bg-dark-accent-yellow text-dark-bg rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors">
+          Download failures
+        </button>
+
+        <!-- Report Image -->
+        <img :src="reportImageUrl" alt="Generated Plot" v-if="reportImageUrl"
+          loading="lazy" class="mx-auto rounded-lg shadow-lg">
+
+        <!-- Logs Section -->
+        <section class="bg-dark-card rounded-lg p-6 shadow-lg mt-8">
+          <div @click="toggleLogs"
+            class="flex justify-between items-center cursor-pointer">
+            <h2 class="text-2xl font-bold">Logs</h2>
+            <svg :class="{'rotate-180': showLogs}"
+              class="w-6 h-6 transition-transform duration-200"
+              xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none"
+              stroke="currentColor" stroke-width="2" stroke-linecap="round"
+              stroke-linejoin="round">
+              <polyline points="6 9 12 15 18 9"></polyline>
+            </svg>
+          </div>
+
+          <div v-show="showLogs" class="mt-4">
+            <div class="mb-4 flex justify-between items-center">
+              <span class="text-sm text-gray-400">Showing latest {{
+                Math.min(logs.length, maxDisplayedLogs) }} of {{ logs.length }}
+                logs</span>
+              <button @click="downloadLogs"
+                class="bg-dark-accent-green text-dark-bg rounded-lg px-4 py-2 text-sm font-medium hover:bg-opacity-80 transition-colors">
+                Download Logs
+              </button>
+            </div>
+            <div class="bg-dark-bg p-4 rounded-lg max-h-96 overflow-y-auto">
+              <div v-for="(log, index) in displayedLogs" :key="index"
+                class="mb-2 last:mb-0">
+                <span class="text-dark-accent-green">{{ log.timestamp }}</span>
+                <span class="ml-2"
+                  :class="{'text-dark-accent-red': log.level === 'ERROR'}">{{
+                  log.message }}</span>
+              </div>
+            </div>
+          </div>
+        </section>
+      </main>
+</template>
+<script>
+import { LLM_CONFIGS, LLM_SPECS,has_image, has_files, _getFailureRateColor, _getFailureRateScore,URL } from '../../public/base.js';
+import { ref, useTemplateRef, onMounted } from 'vue'
+
+const textarea= useTemplateRef('textarea')
+export default{
+    name: 'PageConfigs',
+    data(){
+        return {
+        progressWidth: '0%',
+        modelSpec: LLM_SPECS[0],
+        budget: 50,
+        isFocused: false, // Tracks if the textarea is focused
+        showParams: false,
+        showResetConfirmation: false,
+        enableChartDiagram: true,
+        enableLogging: false,
+        enableConcurrency: false,
+        optimize: false,
+        enableMultiStepAttack: false,
+        scanResults: [],
+        mainTable: [],
+        integrationVerified: false,
+        scanRunning: false,
+        errorMsg: '',
+        maskMode: false,
+        okMsg: '',
+        reportImageUrl: '',
+        selectedConfig: 0,
+        showModules: false,
+        showLogs: false,
+        showConsentModal: true,
+        statusDotClass: 'bg-gray-500', // Default status dot class
+        statusText: 'Verified', // Default status text
+        statusClass: 'bg-green-500 text-dark-bg', // Default status class
+        showLLMSpec: true, // Default to showing the LLM Spec Input
+        logs: [], // This will store all the logs
+        maxDisplayedLogs: 50, // Maximum number of logs to display
+        configs: LLM_CONFIGS,
+        dataConfig: [],
+    }
+    },   
+    created() {
+        // Check if consent is already given in local storage
+        const consentGiven = localStorage.getItem('consentGiven');
+        if (consentGiven === 'true') {
+            this.showConsentModal = false; // Don't show the modal if consent was given
+        }
+    },
+    mounted: function () {
+        this.adjustHeight({ target: this.$refs.textarea }); 
+        // this.startScan();
+        this.loadConfigs();
+
+    },
+    computed: {
+        selectedDS: function () {
+            return this.dataConfig.filter(p => p.selected).length;
+        },
+        displayedLogs() {
+            return this.logs.slice(-this.maxDisplayedLogs).reverse();
+        },
+        hasImageSpec() {
+            return has_image(this.modelSpec);
+        },
+        hasAudioSpec() {
+            return has_files(this.modelSpec);
+        },
+        hasFileSpec() {
+            return has_files(this.modelSpec) || has_image(this.modelSpec);
+        },
+        highlightedText() {
+            // First highlight <<VAR>> pattern
+            let text = this.modelSpec.replace(
+                /<<([^>]+)>>/g,
+                `<span class="px-2 py-0.5 rounded-full bg-dark-accent-yellow text-dark-bg font-medium">&lt;&lt;$1&gt;&gt;</span>`
+            );
+
+            // Then highlight $VARIABLE pattern
+            text = text.replace(
+                /(\$[A-Z_]+)/g,
+                `<span class="px-2 py-0.5 rounded-full bg-yellow-100 text-dark-bg font-medium">$1</span>`
+            );
+
+            // Finally wrap everything in gray text
+            return `<span class="text-gray-500">${text}</span>`;
+        },
+        highlightedText2() {
+            // First apply the highlighting for variables
+            const highlightedText = this.modelSpec.replace(
+                /<<([^>]+)>>/g,
+                `<span class="px-2 py-0.5 rounded-full bg-dark-accent-yellow text-dark-bg font-medium">&lt;&lt;$1&gt;&gt;</span>`
+            );
+
+            // Wrap the entire text in a span to make non-highlighted parts dim gray
+            return `<span class="text-gray-500">${highlightedText}</span>`;
+        }
+
+    },
+    methods: {
+        focusTextarea() {
+            this.isFocused = true;
+            self = this.$refs;
+            this.$nextTick(() => {
+                // Focus the textarea after rendering
+                this.$refs.textarea?.focus();
+                this.adjustHeight({ target: this.$refs.textarea });
+            });
+            document.addEventListener("mousedown", this.handleClickOutside);
+
+        },
+        handleOutsideClick(event) {
+            if (!this.$refs.container.contains(event.target)) {
+                this.isFocused = false;
+                document.removeEventListener("mousedown", this.handleClickOutside);
+            }
+        },
+        unfocusTextarea() {
+            this.isFocused = false;
+        },
+        acceptConsent() {
+            this.showConsentModal = false; // Close the modal
+            localStorage.setItem('consentGiven', 'true'); // Save consent to local storage
+        },
+
+        saveStateToLocalStorage() {
+            const state = {
+                modelSpec: this.modelSpec,
+                budget: this.budget,
+                dataConfig: this.dataConfig,
+                optimize: this.optimize,
+                enableChartDiagram: this.enableChartDiagram,
+                enableMultiStepAttack: this.enableMultiStepAttack,
+            };
+            localStorage.setItem('appState:v1', JSON.stringify(state));
+        },
+        loadStateFromLocalStorage() {
+            const savedState = localStorage.getItem('appState:v1');
+            console.log('Loading state from local storage:', savedState);
+            if (savedState) {
+                const state = JSON.parse(savedState);
+                this.modelSpec = state.modelSpec;
+                this.budget = state.budget;
+                this.dataConfig = state.dataConfig;
+                this.optimize = state.optimize;
+                this.enableChartDiagram = state.enableChartDiagram;
+                this.enableMultiStepAttack = state.enableMultiStepAttack;
+            }
+        },
+        resetState() {
+            localStorage.removeItem('appState:v1');
+            this.modelSpec = LLM_SPECS[0];
+            this.budget = 50;
+            this.dataConfig.forEach(config => config.selected = false);
+            this.optimize = false;
+            this.enableChartDiagram = true;
+            this.okMsg = '';
+            this.errorMsg = '';
+            this.integrationVerified = false;
+            this.showResetConfirmation = false;
+            this.enableMultiStepAttack = false;
+        },
+        confirmResetState() {
+            this.showResetConfirmation = true;
+        },
+        updateStatusDot(ok) {
+            if (ok) {
+                this.statusDotClass = 'bg-green-500'; // Green when expanded
+            } else if (!ok) {
+                this.statusDotClass = 'bg-orange-500'; // Orange if collapsed with content
+            } else {
+                this.statusDotClass = 'bg-gray-500'; // Gray if collapsed without content
+            }
+        },
+        toggleLLMSpec() {
+            this.showLLMSpec = !this.showLLMSpec;
+        },
+        // adjustHeight(event) {
+        //   console.log(event,"event")
+        //     const textarea = event.target;
+        //     event.target.style.height = 'auto';
+        //     event.target.style.height = event.target.scrollHeight + 'px';
+        // },
+        downloadFailures() {
+            window.open('/failures', '_blank');
+        },
+        hide() {
+            this.maskMode = !this.maskMode;
+        },
+        verifyIntegration: async function () {
+            let payload = {
+                spec: this.modelSpec,
+            };
+            const response = await fetch(`${URL}/verify`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(payload),
+            });
+            console.log(response);
+            let txt = await response.text();
+            if (!response.ok) {
+                this.updateStatusDot(false);
+                this.errorMsg = 'Integration verification failed:' + txt;
+            } else {
+                this.errorMsg = '';
+                this.updateStatusDot(true);
+                this.okMsg = 'Integration verified';
+                this.integrationVerified = true;
+                // console.log('Integration verified', this.integrationVerified);
+                // this.$forceUpdate();
+
+            }
+            this.saveStateToLocalStorage();
+        },
+        loadConfigs: async function () {
+            const response = await fetch(`${URL}/v1/data-config`, {
+                method: 'GET',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+            });
+            console.log(response);
+            this.dataConfig = await response?.json();
+            this.loadStateFromLocalStorage();
+        },
+        selectConfig(index) {
+            this.selectedConfig = index;
+            this.modelSpec = LLM_SPECS[index];
+            this.adjustHeight({ target: this.$refs.textarea });
+            // this.adjustHeight({ target: document.getElementById('llm-spec') });
+            this.errorMsg = '';
+            this.okMsg = '';
+            this.integrationVerified = false;
+        },
+        toggleModules() {
+            this.showModules = !this.showModules;
+        },
+        toggleLogs() {
+            this.showLogs = !this.showLogs;
+        },
+        addLog(message, level = 'INFO') {
+            const timestamp = new Date().toISOString();
+            this.logs.push({ timestamp, message, level });
+        },
+        downloadLogs() {
+            const logText = this.logs.map(log => `${log.timestamp} [${log.level}] ${log.message}`).join('\n');
+            const blob = new Blob([logText], { type: 'text/plain' });
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = 'vulnerability_scan_logs.txt';
+            document.body.appendChild(a);
+            a.click();
+            document.body.removeChild(a);
+            URL.revokeObjectURL(url);
+        },
+        addPackage(index) {
+
+             const pkg = this.dataConfig[index];
+            pkg.selected = !pkg.selected;
+
+        },
+        getFailureRateScore(failureRate) {
+            return _getFailureRateScore(failureRate);
+        },
+        getFailureRateColor(failureRate) {
+            return _getFailureRateColor(failureRate);
+        },
+        toggleParams() {
+            this.showParams = !this.showParams;
+        },
+        adjustHeight(event) {
+            const element = event.target;
+            if (!element) {
+                return
+            }
+            // Reset height to ensure accurate measurement
+            element.style.height = 'auto';
+            // Adjust height based on scrollHeight
+            element.style.height = `${element.scrollHeight + 100}px`;
+        },
+        newEvent: function (event) {
+
+            if (event.status) {
+                this.okMsg = `${event.module}`;
+                return
+            }
+            console.log('New event');
+            //  { "module": "Module 49", "tokens": 480, "cost": 4.800000000000001, "progress": 9.8 }
+            let progress = event.progress;
+            progress = progress % 100;
+            this.progressWidth = `${progress}%`;
+            this.addLog(`${JSON.stringify(event)}`, 'INFO');
+            if (this.mainTable.length < 1) {
+                this.mainTable.push(event);
+                event.last = true;
+
+                return
+            }
+            let last = this.mainTable[this.mainTable.length - 1];
+            if (last.module === event.module) {
+                last.tokens = event.tokens;
+                last.cost = event.cost;
+                last.progress = event.progress;
+                last.failureRate = event.failureRate;
+            } else {
+                last.last = false;
+                this.mainTable.push(event);
+                event.last = true;
+                this.newRow()
+            }
+            this.okMsg = `New event: ${event.module}: ${event.progress}%`;
+
+        },
+        newRow: async function () {
+            if (!this.enableChartDiagram) {
+                return
+            }
+            console.log('New row');
+            let payload = {
+                table: this.mainTable,
+            };
+            const response = await fetch(`${URL}/plot.jpeg`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(payload),
+            });
+            // Convert image response to a data URL for the <img> src
+            const blob = await response.blob();
+            const reader = new FileReader();
+            reader.readAsDataURL(blob);
+            reader.onloadend = () => {
+                this.reportImageUrl = reader.result;
+            };
+        },
+        selectAllPackages() {
+            const allSelected = this.dataConfig.every(pkg => pkg.selected);
+
+            // If all are selected, deselect all. Otherwise, select all.
+            this.dataConfig.forEach(pkg => {
+                pkg.selected = !allSelected;
+            });
+
+            this.updateSelectedDS();
+        },
+
+        deselectAllPackages() {
+            this.dataConfig.forEach(pkg => {
+                pkg.selected = false;
+            });
+            this.updateSelectedDS();
+        },
+
+        updateSelectedDS() {
+            this.selectedDS = this.dataConfig.filter(pkg => pkg.selected).length;
+        },
+        updateBudgetFromSlider(event) {
+            this.budget = parseInt(event.target.value);
+        },
+        updateBudgetFromInput(event) {
+            let value = parseInt(event.target.value);
+            if (isNaN(value) || value < 1) {
+                value = 1;
+            } else if (value > 100) {
+                value = 100;
+            }
+            this.budget = value;
+        },
+        stopScan: async function () {
+            this.scanRunning = false;
+            const response = await fetch(`${URL}/stop`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+            });
+        },
+        startScan: async function () {
+            this.showLLMSpec = false;
+            let payload = {
+                maxBudget: this.budget,
+                llmSpec: this.modelSpec,
+                datasets: this.dataConfig,
+                optimize: this.optimize,
+                enableMultiStepAttack: this.enableMultiStepAttack,
+            };
+            const response = await fetch(`${URL}/scan`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(payload),
+            });
+            this.okMsg = 'Scan started';
+            this.mainTable = [];
+            this.scanRunning = true;
+            const reader = response.body.getReader();
+            let receivedLength = 0; // received that many bytes at the moment
+            let chunks = []; // array of received binary chunks (comprises the body)
+            while (true) {
+                const { done, value } = await reader.read();
+
+                if (done) {
+                    break;
+                }
+
+                chunks.push(value);
+                receivedLength += value.length;
+
+                const chunkAsString = new TextDecoder("utf-8").decode(value);
+                const chunkAsLines = chunkAsString.split('\n').filter(line => line.trim());
+
+                self = this;
+                chunkAsLines.forEach(line => {
+                    try {
+                        const result = JSON.parse(line);
+                        self.scanResults.push(result);
+                        self.newEvent(result);
+                    } catch (e) {
+                        console.error('Error parsing chunk:', e);
+                    }
+                });
+            }
+            this.saveStateToLocalStorage();
+
+        }
+    }
+}
+</script>

ui/src/components/PageContent.vue

@@ -0,0 +1,103 @@
+<template>
+   <div id="consent-modal" v-if="showConsentModal"
+    class="fixed inset-0 bg-black bg-opacity-75 flex justify-center items-center z-50">
+    <div
+        class="bg-dark-card text-dark-text p-8 rounded-xl shadow-2xl max-w-xl w-full">
+        <h2 class="text-2xl font-bold mb-6 text-center">AI Red Team Ethical
+            Use Agreement</h2>
+        <div class="space-y-6">
+            <p class="text-sm leading-relaxed">
+                This AI red team tool is designed for security research,
+                vulnerability assessment,
+                and responsible testing purposes. By accessing this tool, you
+                explicitly agree to
+                the following ethical guidelines:
+            </p>
+            <ul class="list-disc list-inside text-sm space-y-3">
+                <li>
+                    <strong>Consent and Authorization:</strong> You will only
+                    use
+                    this tool on systems
+                    for which you have explicit, documented permission from the
+                    system owners.
+                </li>
+                <li>
+                    <strong>Responsible Disclosure:</strong> Any vulnerabilities
+                    discovered must be
+                    reported responsibly to the appropriate parties,
+                    prioritizing
+                    system and user safety.
+                </li>
+                <li>
+                    <strong>No Malicious Intent:</strong> You will not use this
+                    tool
+                    to cause harm,
+                    disrupt services, or compromise the integrity of any system
+                    or
+                    data.
+                </li>
+                <li>
+                    <strong>Legal Compliance:</strong> All testing and research
+                    must
+                    comply with
+                    applicable local, national, and international laws and
+                    regulations.
+                </li>
+
+            </ul>
+
+            <p class="text-xs text-gray-400 italic">
+                Violation of these terms may result in immediate termination of
+                access and
+                potential legal consequences.
+            </p>
+        </div>
+        <div class="flex justify-center space-x-4 mt-8">
+            <button
+                @click="declineConsent"
+                class="bg-dark-accent-red text-white rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors">
+                Decline
+            </button>
+            <button
+                @click="acceptConsent"
+                class="bg-dark-accent-green text-dark-bg rounded-lg px-6 py-3 font-medium hover:bg-opacity-80 transition-colors">
+                I Agree and Understand
+            </button>
+        </div>
+    </div>
+    
+</div>
+
+
+</template>
+
+<script>
+export default {
+  name: 'PageContent',
+
+  data() {
+    return {
+      showConsentModal: true // Default to true
+    };
+  },
+
+  emits: ['accept', 'decline'], // Define the custom events
+
+  methods: {
+     acceptConsent() {
+            this.showConsentModal = false; // Close the modal
+            localStorage.setItem('consentGiven', 'true'); // Save consent to local storage
+        },
+    declineConsent() {
+            this.showConsentModal = false; // Close the modal
+            localStorage.setItem('consentGiven', 'false'); // Save decline to local storage
+            window.location.href = 'https://www.google.com'; // Redirect to Google
+        },
+  }
+};
+</script>
+
+
+<style >
+/* Styles for the consent modal */
+</style>

ui/src/components/PageFooter.vue

@@ -0,0 +1,64 @@
+<template>
+  <footer class="mt-16 pt-8 border-t border-gray-800">
+    <div class="max-w-6xl mx-auto px-4 sm:px-6 lg:px-8">
+      <div class="grid grid-cols-1 md:grid-cols-3 gap-8">
+        <div>
+          <h3 class="text-lg font-semibold text-dark-accent-green mb-4">
+            Home
+          </h3>
+          <p class="text-gray-400">Dedicated to LLM Security, 2025</p>
+        </div>
+
+        <div>
+          <h3 class="text-lg font-semibold text-dark-accent-green mb-4">
+            Connect
+          </h3>
+          <ul class="space-y-2">
+            <li>
+              <a
+                href="https://x.com"
+                target="_blank"
+                rel="noopener noreferrer"
+                class="text-gray-400 hover:text-dark-accent-green"
+                >X.com</a
+              >
+            </li>
+            <li>
+              <a
+                href="https://github.com/msoedov"
+                target="_blank"
+                rel="noopener noreferrer"
+                class="text-gray-400 hover:text-dark-accent-green"
+                >Github</a
+              >
+            </li>
+          </ul>
+        </div>
+
+        <div>
+          <h3 class="text-lg font-semibold text-dark-accent-green mb-4">
+            About
+          </h3>
+          <p class="text-gray-400">
+            This is the LLM Vulnerability Scanner. Easy to use—no coding needed,
+            just pure security testing.
+          </p>
+        </div>
+      </div>
+
+      <div class="mt-8 pt-8 border-t border-gray-800 text-center">
+        <p class="text-gray-400">Made with ❤️ by the Agentic Security Team</p>
+      </div>
+    </div>
+  </footer>
+</template>
+
+<script>
+export default {
+  name: "PageFooter", // Descriptive name
+};
+</script>
+
+<style scoped>
+/* Footer-specific styles here */
+</style>

ui/src/components/PageHeader.vue

@@ -0,0 +1,22 @@
+<template>
+  <div>hello</div>
+</template>
+
+<script>
+export default {
+  name: 'PageHeader', // Give a descriptive name
+  // No specific JavaScript logic needed for this simple header
+  // You can add props if you want to make the title dynamic:
+  props: {
+    title: {
+      type: String,
+      default: 'LLM Vulnerability Scanner' // Default title
+    }
+  }
+};
+</script>
+
+<style scoped>
+/* Any header-specific styles can go here */
+/* If you are using tailwind, you can include this as well*/
+</style>

ui/src/main.js

@@ -0,0 +1,11 @@
+import { createApp } from 'vue'
+import App from './App.vue' // Create App.vue (see next step)
+import '../public/base.js' // If you have this file, move it to src/assets
+import '../public/telemetry.js' // Move to src/assets
+import lucide from 'lucide' // Import lucide if you are using it
+const app = createApp(App)
+app.mount('#vue-app') // Change #vue-app to #app
+
+app.config.globalProperties.$lucide = lucide
+
+//lucide.createIcons(); // Create icons

ui/tailwind.config.js

@@ -0,0 +1,30 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: ["./src/**/*.{vue,js,ts,jsx,tsx}"],
+  darkMode: 'class',
+  theme: {
+    extend: {
+      fontFamily: {
+        sans: ['Inter', 'sans-serif'],
+        technopollas: ['Technopollas', 'sans-serif'],
+      },
+      colors: {
+        dark: {
+          bg: '#121212',
+          card: '#1E1E1E',
+          text: '#FFFFFF',
+          accent: {
+            green: '#4CAF50',
+            red: '#F44336',
+            orange: '#FF9800',
+            yellow: '#FFEB3B',
+          },
+        },
+      },
+      borderRadius: {
+        'lg': '1rem',
+      },
+    }
+  },
+  plugins: [],
+}

ui/vue.config.js

@@ -0,0 +1,2 @@
+const { defineConfig } = require('@vue/cli-service')
+module.exports = defineConfig({ transpileDependencies: true, publicPath: '/' ,devServer: { allowedHosts: 'all', client: {webSocketURL: 'auto://0.0.0.0:0/ws'}}, })