build(deps-dev): bump setuptools from 69.5.1 to 70.0.0

Bumps [setuptools](https://github.com/pypa/setuptools) from 69.5.1 to 70.0.0.
- [Release notes](https://github.com/pypa/setuptools/releases)
- [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
- [Commits](https://github.com/pypa/setuptools/compare/v69.5.1...v70.0.0)

---
updated-dependencies:
- dependency-name: setuptools
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

.gitignore

@@ -5,3 +5,4 @@ __pycache__/
 failures.csv
 runs/
 *.todo
+logs/

Readme.md

@@ -26,12 +26,16 @@
 - LLM API integration and stress testing 🛠️
 - Wide range of fuzzing and attack techniques 🌀
 
+| Tool                    | Source                                                                        | Integrated |
+|-------------------------|-------------------------------------------------------------------------------|------------|
+| Garak                   | [leondz/garak](https://github.com/leondz/garak)                               | ✅          |
+| InspectAI               | [UKGovernmentBEIS/inspect_ai](https://github.com/UKGovernmentBEIS/inspect_ai) | ✅          |
+| llm-adaptive-attacks    | [tml-epfl/llm-adaptive-attacks](https://github.com/tml-epfl/llm-adaptive-attacks) | ✅       |
+| Custom Huggingface Datasets | markush1/LLM-Jailbreak-Classifier                                                                         | ✅          |
+| Local CSV Datasets      | -                                                                             | ✅          |
+
 Note: Please be aware that Agentic Security is designed as a safety scanner tool and not a foolproof solution. It cannot guarantee complete protection against all possible threats.
 
-## About the Project 🧙
-
-<img width="100%" alt="booking-screen" src="https://res.cloudinary.com/do9qa2bqr/image/upload/v1713002396/1-ezgif.com-video-to-gif-converter_s2hsro.gif">
-
 ## 📦 Installation
 
 To get started with Agentic Security, simply install the package using pip:
@@ -63,6 +67,10 @@ agentic_security --port=PORT --host=HOST
 
 ```
 
+## UI 🧙
+
+<img width="100%" alt="booking-screen" src="https://res.cloudinary.com/do9qa2bqr/image/upload/v1713002396/1-ezgif.com-video-to-gif-converter_s2hsro.gif">
+
 ## LLM kwargs
 
 Agentic Security uses plain text HTTP spec like:
@@ -292,11 +300,7 @@ Agentic Security is released under the Apache License v2.
 
 ## Contact us
 
-## 🤝 Schedule a 1-on-1 Session
 
-<a href="https://cal.com/alexander-myasoedov-go2tfs/30min"><img src="https://cal.com/book-with-cal-dark.svg" alt="Book us with Cal.com"></a>
-
-Book a 1-on-1 Session with the founders, to discuss any issues, provide feedback, or explore how we can improve agentic_security for you.
 
 ## Repo Activity
 

agentic_security/app.py

@@ -1,14 +1,15 @@
 import random
-import sys
 from asyncio import Event, Queue
 from datetime import datetime
+from logging import config
 from pathlib import Path
 
-from fastapi import BackgroundTasks, FastAPI, HTTPException, Response
+from fastapi import BackgroundTasks, FastAPI, HTTPException, Request, Response
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse, StreamingResponse
 from loguru import logger
 from pydantic import BaseModel
+from starlette.middleware.base import BaseHTTPMiddleware
 
 from .http_spec import LLMSpec
 from .probe_actor import fuzzer
@@ -16,15 +17,6 @@ from .probe_actor.refusal import REFUSAL_MARKS
 from .probe_data import REGISTRY
 from .report_chart import plot_security_report
 
-logger.remove(0)
-logger.add(
-    sys.stderr,
-    format="<green>[{level}]</green> <blue>{time:YYYY-MM-DD HH:mm:ss.SS}</blue> | <cyan>{module}:{function}:{line}</cyan> | <white>{message}</white>",
-    colorize=True,
-    level="INFO",
-)
-
-
 # Create the FastAPI app instance
 app = FastAPI()
 origins = [
@@ -164,13 +156,13 @@ class Message(BaseModel):
 class CompletionRequest(BaseModel):
     model: str
     messages: list[Message]
-    temperature: float
-    top_p: float
-    n: int
-    stop: list[str]
-    max_tokens: int
-    presence_penalty: float
-    frequency_penalty: float
+    temperature: float = 0.7  # Default value for temperature
+    top_p: float = 1.0  # Default value for top_p
+    n: int = 1  # Default value for n
+    stop: list[str] = None  # Optional; specify as None if not provided
+    max_tokens: int = 100  # Default value for max_tokens
+    presence_penalty: float = 0.0  # Default value for presence_penalty
+    frequency_penalty: float = 0.0  # Default value for frequency_penalty
 
 
 # OpenAI proxy endpoint
@@ -206,3 +198,41 @@ async def proxy_completions(request: CompletionRequest):
             }
         ],
     }
+
+
+config.dictConfig(
+    {
+        "version": 1,
+        "disable_existing_loggers": True,
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+            },
+        },
+        "root": {
+            "handlers": ["console"],
+            "level": "INFO",
+        },
+        "loggers": {
+            "uvicorn.access": {
+                "level": "ERROR",  # Set higher log level to suppress info logs globally
+                "handlers": ["console"],
+                "propagate": False,
+            }
+        },
+    }
+)
+
+
+class LogNon200ResponsesMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        response = await call_next(request)
+        if response.status_code != 200:
+            logger.error(
+                f"{request.method} {request.url} - Status code: {response.status_code}"
+            )
+        return response
+
+
+# Add middleware to the application
+app.add_middleware(LogNon200ResponsesMiddleware)

agentic_security/probe_data/__init__.py

@@ -7,7 +7,7 @@ REGISTRY = [
         "tokens": 224196,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
-        "selected": True,
+        "selected": False,
         "dynamic": False,
         "url": "https://huggingface.co/ShawnMenz/DAN_jailbreak",
     },
@@ -17,7 +17,7 @@ REGISTRY = [
         "tokens": 6988,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
-        "selected": True,
+        "selected": False,
         "dynamic": False,
         "url": "https://huggingface.co/deepset/prompt-injections",
     },
@@ -27,7 +27,7 @@ REGISTRY = [
         "tokens": 26971,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
-        "selected": True,
+        "selected": False,
         "dynamic": False,
         "url": "https://huggingface.co/rubend18/ChatGPT-Jailbreak-Prompts",
     },
@@ -37,7 +37,7 @@ REGISTRY = [
         "tokens": 7172,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
-        "selected": True,
+        "selected": False,
         "dynamic": False,
         "url": "https://huggingface.co/notrichardren/refuse-to-answer-prompts",
     },
@@ -47,7 +47,7 @@ REGISTRY = [
         "tokens": 19758,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
-        "selected": True,
+        "selected": False,
         "dynamic": False,
         "url": "https://huggingface.co/Lemhf14/EasyJailbreak_Datasets",
     },
@@ -57,17 +57,37 @@ REGISTRY = [
         "tokens": 19758,
         "approx_cost": 0.0,
         "source": "Hugging Face Datasets",
-        "selected": True,
+        "selected": False,
         "dynamic": False,
         "url": "https://huggingface.co/markush1/LLM-Jailbreak-Classifier",
     },
+    {
+        "dataset_name": "JailbreakV-28K/JailBreakV-28k",
+        "num_prompts": 28300,
+        "tokens": 1975800,
+        "approx_cost": 0.0,
+        "source": "Hugging Face Datasets",
+        "selected": True,
+        "dynamic": False,
+        "url": "https://huggingface.co/JailbreakV-28K/JailBreakV-28k",
+    },
+    {
+        "dataset_name": "ShawnMenz/jailbreak_sft_rm_ds",
+        "num_prompts": 371000,
+        "tokens": 1975800,
+        "approx_cost": 0.0,
+        "source": "Hugging Face Datasets",
+        "selected": False,
+        "dynamic": False,
+        "url": "https://huggingface.co/ShawnMenz/jailbreak_sft_rm_ds",
+    },
     {
         "dataset_name": "Steganography",
         "num_prompts": 10,
         "tokens": 0,
         "approx_cost": 0.0,
         "source": "Local mutation dataset",
-        "selected": True,
+        "selected": False,
         "dynamic": True,
         "url": "",
     },
@@ -77,7 +97,7 @@ REGISTRY = [
         "tokens": 0,
         "approx_cost": 0.0,
         "source": "Local mutation dataset",
-        "selected": True,
+        "selected": False,
         "dynamic": True,
         "url": "",
     },
@@ -87,10 +107,30 @@ REGISTRY = [
         "tokens": 0,
         "approx_cost": 0.0,
         "source": "Local dataset",
-        "selected": True,
+        "selected": False,
         "dynamic": True,
         "url": "",
     },
+    {
+        "dataset_name": "jailbreak_llms/2023_05_07",
+        "num_prompts": 0,
+        "tokens": 0,
+        "approx_cost": 0.0,
+        "source": "Github",
+        "selected": False,
+        "dynamic": True,
+        "url": "https://github.com/verazuo/jailbreak_llms",
+    },
+    {
+        "dataset_name": "jailbreak_llms/2023_12_25.csv",
+        "num_prompts": 0,
+        "tokens": 0,
+        "approx_cost": 0.0,
+        "source": "Github",
+        "selected": False,
+        "dynamic": True,
+        "url": "https://github.com/verazuo/jailbreak_llms",
+    },
     {
         "dataset_name": "Malwaregen",
         "num_prompts": 0,
@@ -141,6 +181,16 @@ REGISTRY = [
         "url": "https://github.com/leondz/garak2",
         "dynamic": True,
     },
+    {
+        "dataset_name": "InspectAI",
+        "num_prompts": 0,
+        "tokens": 0,
+        "approx_cost": 0.0,
+        "source": "Github: https://github.com/UKGovernmentBEIS/inspect_ai",
+        "selected": False,
+        "url": "https://github.com/UKGovernmentBEIS/inspect_ai",
+        "dynamic": True,
+    },
     {
         "dataset_name": "Custom CSV",
         "num_prompts": len(load_local_csv().prompts),

agentic_security/probe_data/data.py

@@ -1,13 +1,19 @@
+import io
 import os
 import random
 from dataclasses import dataclass
 from functools import lru_cache
 
+import httpx
 import pandas as pd
 from loguru import logger
 
 from agentic_security.probe_data import stenography_fn
-from agentic_security.probe_data.modules import adaptive_attacks, garak_tool
+from agentic_security.probe_data.modules import (
+    adaptive_attacks,
+    garak_tool,
+    inspect_ai_tool,
+)
 
 IS_VERCEL = os.getenv("IS_VERCEL", "f") == "t"
 
@@ -144,6 +150,44 @@ def load_dataset_v6():
     )
 
 
+@cache_to_disk()
+def load_dataset_v7():
+
+    splits = {
+        "mini_JailBreakV_28K": "JailBreakV_28K/mini_JailBreakV_28K.csv",
+        "JailBreakV_28K": "JailBreakV_28K/JailBreakV_28K.csv",
+    }
+    df = pd.read_csv(
+        "hf://datasets/JailbreakV-28K/JailBreakV-28k/" + splits["JailBreakV_28K"]
+    )
+    bad_prompts = df["jailbreak_query"].tolist()
+    print(df.shape)
+    return ProbeDataset(
+        dataset_name="JailbreakV-28K/JailBreakV-28k",
+        metadata={},
+        prompts=bad_prompts,
+        tokens=count_words_in_list(bad_prompts),
+        approx_cost=0.0,
+    )
+
+
+@cache_to_disk()
+def load_dataset_v8():
+
+    df = pd.read_csv(
+        "hf://datasets/ShawnMenz/jailbreak_sft_rm_ds/jailbreak_sft_rm_ds.csv",
+        names=["jailbreak", "prompt"],
+    )
+    filtered = df[df["jailbreak"] == "jailbreak"]["prompt"].tolist()
+    return ProbeDataset(
+        dataset_name="JailbreakV-28K/JailBreakV-28k",
+        metadata={},
+        prompts=filtered,
+        tokens=count_words_in_list(filtered),
+        approx_cost=0.0,
+    )
+
+
 @cache_to_disk()
 def load_dataset_v5():
     from datasets import load_dataset
@@ -169,6 +213,22 @@ def load_dataset_v5():
     )
 
 
+@cache_to_disk()
+def load_generic_csv(url, name, column="prompt", predicator=None):
+    r = httpx.get(url)
+    content = r.content
+    df = pd.read_csv(io.StringIO(content.decode("utf-8")))
+    logger.info(f"Loaded {len(df)} prompts from {url}")
+    filtered_prompts = df[df.apply(predicator, axis=1)][column].tolist()
+    return ProbeDataset(
+        dataset_name=name,
+        metadata={},
+        prompts=filtered_prompts,
+        tokens=count_words_in_list(filtered_prompts),
+        approx_cost=0.0,
+    )
+
+
 def prepare_prompts(dataset_names, budget, tools_inbox=None):
     # ## Datasets used and cleaned:
     # markush1/LLM-Jailbreak-Classifier
@@ -184,6 +244,20 @@ def prepare_prompts(dataset_names, budget, tools_inbox=None):
         "rubend18/ChatGPT-Jailbreak-Prompts": load_dataset_v3,
         "Lemhf14/EasyJailbreak_Datasets": load_dataset_v5,
         "markush1/LLM-Jailbreak-Classifier": load_dataset_v6,
+        "JailbreakV-28K/JailBreakV-28k": load_dataset_v7,
+        "ShawnMenz/jailbreak_sft_rm_ds": load_dataset_v8,
+        "verazuo/jailbreak_llms/2023_05_07": lambda: load_generic_csv(
+            url="https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/prompts/jailbreak_prompts_2023_05_07.csv",
+            name="verazuo/jailbreak_llms/2023_05_07",
+            column="prompt",
+            predicator=lambda x: bool(x["jailbreak"]),
+        ),
+        "verazuo/jailbreak_llms/2023_12_25.csv": lambda: load_generic_csv(
+            url="https://raw.githubusercontent.com/verazuo/jailbreak_llms/main/data/prompts/jailbreak_prompts_2023_12_25.csv.csv",
+            name="verazuo/jailbreak_llms/2023_12_25.csv",
+            column="prompt",
+            predicator=lambda x: bool(x["jailbreak"]),
+        ),
         "Custom CSV": load_local_csv,
     }
 
@@ -206,6 +280,11 @@ def prepare_prompts(dataset_names, budget, tools_inbox=None):
             garak_tool.Module(group, tools_inbox=tools_inbox).apply(),
             lazy=True,
         ),
+        "InspectAI": lambda: dataset_from_iterator(
+            "InspectAI",
+            inspect_ai_tool.Module(group, tools_inbox=tools_inbox).apply(),
+            lazy=True,
+        ),
         "GPT fuzzer": lambda: [],
     }
 

agentic_security/probe_data/modules/garak_tool.py

@@ -9,7 +9,6 @@ from loguru import logger
 
 
 class Module:
-
     def __init__(self, prompt_groups: [], tools_inbox: asyncio.Queue):
         self.tools_inbox = tools_inbox
         if not self.is_garak_installed():

agentic_security/probe_data/modules/inspect_ai_task.py

@@ -0,0 +1,13 @@
+from inspect_ai import Task, eval, task
+from inspect_ai.dataset import example_dataset
+from inspect_ai.scorer import model_graded_fact
+from inspect_ai.solver import chain_of_thought, generate, self_critique
+
+
+@task
+def theory_of_mind():
+    return Task(
+        dataset=example_dataset("theory_of_mind"),
+        plan=[chain_of_thought(), generate(), self_critique()],
+        scorer=model_graded_fact(),
+    )

agentic_security/probe_data/modules/inspect_ai_tool.py

@@ -0,0 +1,71 @@
+import asyncio
+import importlib.util
+import os
+
+from loguru import logger
+
+inspect_ai_task = (
+    __file__.replace("inspect_ai_tool.py", "inspect_ai_task.py")
+    .replace(os.getcwd(), "")
+    .strip("/")
+)
+
+
+class Module:
+    name = "Inspect AI"
+
+    def __init__(self, prompt_groups: [], tools_inbox: asyncio.Queue):
+        self.tools_inbox = tools_inbox
+        if not self.is_tool_installed():
+            logger.error(
+                "inspect_ai module is not installed. Please install it using 'pip install inspect_ai'"
+            )
+
+    def is_tool_installed(self) -> bool:
+        inspect_ai = importlib.util.find_spec("inspect_ai")
+        return inspect_ai is not None
+
+    async def _proc(self, command):
+        env = os.environ.copy()
+        env["OPENAI_API_BASE"] = "http://0.0.0.0:8718/proxy"
+        process = await asyncio.create_subprocess_shell(
+            command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+            shell=True,
+        )
+
+        logger.info(f"Started {command}")
+
+        # Read output as it becomes available
+        async for line in process.stdout:
+            logger.info(line.decode().strip())
+
+        # Check for errors
+        err = await process.stderr.read()
+        if err:
+            logger.error(err.decode().strip())
+
+        await process.wait()
+        logger.info(f"Command {command} {process}finished.")
+
+    async def apply(self) -> []:
+        env = os.environ.copy()
+        env["OPENAI_API_BASE"] = "http://0.0.0.0:8718/proxy"
+
+        # Command to be executed
+        command = f"inspect eval {inspect_ai_task} --model openai/gpt-4  --model-base-url=http://0.0.0.0:8718/proxy"
+        logger.info(f"Executing command: {command}")
+
+        proc = asyncio.create_task(self._proc(command))
+        is_empty = self.tools_inbox.empty()
+        await asyncio.sleep(2)
+        logger.info(f"Is inbox empty? {is_empty}")
+        while not self.tools_inbox.empty():
+            ref = self.tools_inbox.get_nowait()
+            message, _, ready = ref["message"], ref["reply"], ref["ready"]
+            yield message
+            ready.set()
+        logger.info(f"{self.name} tool finished.")
+        await proc

agentic_security/probe_data/test_data.py

@@ -12,13 +12,13 @@ class TestPreparePrompts:
         # Assert that the prepared_prompts list is empty
         assert prepared_prompts == []
 
-        assert len(
-            prepare_prompts(["markush1/LLM-Jailbreak-Classifier"], 100)
-        ) == snapshot(1)
+        # assert len(
+        #     prepare_prompts(["markush1/LLM-Jailbreak-Classifier"], 100)
+        # ) == snapshot(1)
 
         assert len(
             prepare_prompts(
-                ["markush1/LLM-Jailbreak-Classifier", "llm-adaptive-attacks"],
+                ["llm-adaptive-attacks"],
                 100,
             )
-        ) == snapshot(2)
+        ) == snapshot(1)

agentic_security/static/index.html

@@ -143,89 +143,98 @@
                   v-model="modelSpec"
                   @input="adjustHeight"></textarea>
               </div>
-              <div class="grid gap-1.5">
-                <label
-                  class="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
-                  for="max-budget">
-                  Maximum Budget in {{budget}}M Tokens
-                </label>
-                <input
-                  class="flex h-10 w-full rounded-md border border-earth-disabled bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
-                  id="max-budget"
-                  placeholder="Enter maximum budget..."
-                  type="number"
-                  v-model="budget" />
-              </div>
-              <div
-                class="rounded-lg text-card-foreground shadow-sm mt-10 mb-10 border border-gray-300">
-                <div class="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 mt-5 mb-5">
-                  <div class="flex flex-col space-y-4">
-                    <!-- Accordion Header -->
-                    <button
-                      @click="toggleDatasets"
-                      class="flex justify-between items-center text-lg font-semibold w-full py-2 text-center">
-                      Modules [{{selectedDS}}]
-                      selected
-                      <svg
-                        :class="{'rotate-180': showDatasets}"
-                        class="h-5 w-5 transform transition-transform duration-200"
-                        xmlns="http://www.w3.org/2000/svg"
-                        fill="none"
-                        viewBox="0 0 24 24"
-                        stroke="currentColor">
-                        <path
-                          stroke-linecap="round"
-                          stroke-linejoin="round"
-                          stroke-width="2"
-                          d="M19 9l-7 7-7-7" />
-                      </svg>
-                    </button>
 
-                    <!-- Accordion Content -->
+              <div class="space-y-4">
+                <div class="flex justify-between items-center">
+                  <label for="max-budget"
+                    class="text-sm font-medium text-gray-700">
+                    Maximum Budget
+                  </label>
+                  <div class="flex items-center space-x-2">
+                    <input
+                      id="budget-display"
+                      v-model="budget"
+                      @change="updateBudgetFromInput"
+                      class="w-20 px-2 py-1 text-right text-sm font-medium text-gray-900 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-indigo-500"
+                      type="text" />
+                    <span class="text-sm font-medium text-gray-600">M
+                      Tokens</span>
+                  </div>
+                </div>
+
+                <div class="relative">
+                  <input
+                    id="max-budget"
+                    v-model="budget"
+                    @input="updateBudgetFromSlider"
+                    type="range"
+                    min="1"
+                    max="100"
+                    step="1"
+                    class="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer">
+                  <div
+                    class="absolute -top-6 left-0 w-full flex justify-between text-xs text-gray-600">
+                    <span>1M</span>
+                    <span>25M</span>
+                    <span>50M</span>
+                    <span>75M</span>
+                    <span>100M</span>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Modules Selection -->
+              <div class="border border-gray-200 rounded-md">
+                <button
+                  @click="toggleDatasets"
+                  class="flex justify-between items-center w-full px-4 py-3 text-left text-gray-700 font-medium bg-gray-50 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-inset">
+                  <span>Modules [{{selectedDS}} selected]</span>
+                  <svg :class="{'rotate-180': showDatasets}"
+                    class="h-5 w-5 text-gray-500"
+                    xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"
+                    fill="currentColor">
+                    <path fill-rule="evenodd"
+                      d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1 0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z"
+                      clip-rule="evenodd" />
+                  </svg>
+                </button>
+                <div v-show="showDatasets" class="p-4 bg-white">
+                  <div class="flex justify-between mb-4">
+                    <button
+                      @click="selectAllPackages"
+                      class="px-3 py-1 text-sm font-medium text-indigo-600 hover:text-indigo-500 focus:outline-none focus:underline">
+                      Select All
+                    </button>
+                    <button
+                      @click="deselectAllPackages"
+                      class="px-3 py-1 text-sm font-medium text-gray-600 hover:text-gray-500 focus:outline-none focus:underline">
+                      Deselect All
+                    </button>
+                  </div>
+                  <div
+                    class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
                     <div
-                      v-show="showDatasets"
-                      class="grid grid-cols-1 md:grid-cols-4 gap-4 transition-all duration-500 ">
-                      <div
-                        v-for="(package, index) in dataConfig"
-                        :key="index"
-                        @click="addPackage(index)"
-                        class="border-2 rounded-lg p-4 flex flex-col items-start hover:shadow-md transition-all"
-                        :class="{'border-earth-1': package.selected, 'border-gray-200': !package.selected}">
-                        <div class="flex items-center justify-between w-full">
-                          <div
-                            class="font-medium"
-                            :class="{'text-earth-1': package.selected, 'text-gray-800': !package.selected}">
-                            {{ package.dataset_name }}
-                          </div>
-                          <svg
-                            class="h-5 w-5"
-                            fill="none"
-                            viewBox="0 0 24 24"
-                            stroke="currentColor"
-                            :class="{'text-earth-1': package.selected, 'text-gray-600': !package.selected}">
-                            <path
-                              stroke-linecap="round"
-                              stroke-linejoin="round"
-                              stroke-width="2"
-                              d="M5 13l4 4L19 7" />
-                          </svg>
-                        </div>
-                        <div class="text-sm text-gray-600">
-                          {{ package.source || 'Local dataset' }}
-                        </div>
-                        <div class="mt-2 text-gray-800 font-semibold"
-                          v-if="!package.dynamic">
-                          {{ package.num_prompts.toLocaleString() }} prompts
-                        </div>
-                        <div class="mt-2 text-gray-800 font-semibold"
-                          v-if="package.dynamic">
-                          Dynamic dataset
-                        </div>
+                      v-for="(package, index) in dataConfig"
+                      :key="index"
+                      @click="addPackage(index)"
+                      class="border rounded-md p-3 cursor-pointer transition-all hover:shadow-md"
+                      :class="{'border-indigo-500 bg-indigo-50': package.selected, 'border-gray-200': !package.selected}">
+                      <div class="font-medium"
+                        :class="{'text-indigo-700': package.selected, 'text-gray-900': !package.selected}">
+                        {{ package.dataset_name }}
+                      </div>
+                      <div class="text-sm text-gray-500 mt-1">{{ package.source
+                        || 'Local dataset' }}</div>
+                      <div class="mt-2 text-sm font-semibold"
+                        :class="{'text-indigo-600': package.selected, 'text-gray-700': !package.selected}">
+                        {{ package.dynamic ? 'Dynamic dataset' :
+                        `${package.num_prompts.toLocaleString()} prompts` }}
                       </div>
                     </div>
                   </div>
                 </div>
               </div>
+
               <div
                 class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative"
                 role="alert" v-if="errorMsg">
@@ -275,6 +284,7 @@
                       d="M5 12h14"></path><path
                       d="m12 5 7 7-7 7"></path></svg>
                   Run Scan
+                  <span class="sr-only">(Initiates the security scan)</span>
                 </button>
               </div>
             </div>
@@ -589,6 +599,35 @@ Content-Type: application/json
               this.reportImageUrl = reader.result;
             };
           },
+          selectAllPackages() {
+            this.dataConfig.forEach(package => {
+              package.selected = true;
+            });
+            this.updateSelectedDS();
+          },
+
+          deselectAllPackages() {
+            this.dataConfig.forEach(package => {
+              package.selected = false;
+            });
+            this.updateSelectedDS();
+          },
+
+          updateSelectedDS() {
+            this.selectedDS = this.dataConfig.filter(package => package.selected).length;
+          },
+          updateBudgetFromSlider(event) {
+            this.budget = parseInt(event.target.value);
+          },
+          updateBudgetFromInput(event) {
+            let value = parseInt(event.target.value);
+            if (isNaN(value) || value < 1) {
+              value = 1;
+            } else if (value > 100) {
+              value = 100;
+            }
+            this.budget = value;
+          },
           startScan: async function() {
             let payload = {
                 maxBudget: this.budget,

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -673,12 +673,12 @@ typing = ["typing-extensions (>=4.8)"]
 
 [[package]]
 name = "fire"
-version = "0.5.0"
+version = "0.6.0"
 description = "A library for automatically generating command line interfaces."
 optional = false
 python-versions = "*"
 files = [
-    {file = "fire-0.5.0.tar.gz", hash = "sha256:a6b0d49e98c8963910021f92bba66f65ab440da2982b78eb1bbf95a0a34aacc6"},
+    {file = "fire-0.6.0.tar.gz", hash = "sha256:54ec5b996ecdd3c0309c800324a0703d6da512241bc73b553db959d98de0aa66"},
 ]
 
 [package.dependencies]
@@ -1087,13 +1087,13 @@ files = [
 
 [[package]]
 name = "inline-snapshot"
-version = "0.8.2"
+version = "0.9.0"
 description = "golden master/snapshot/approval testing library which puts the values right into your source code"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "inline_snapshot-0.8.2-py3-none-any.whl", hash = "sha256:df365176dc04e8054c699981a834e4c4482cb42c34cb3378515e3f65d49a70df"},
-    {file = "inline_snapshot-0.8.2.tar.gz", hash = "sha256:b20515b7bf01675b0f6adaadfd6277ef4456cd797c0735582b07f949a908c2f7"},
+    {file = "inline_snapshot-0.9.0-py3-none-any.whl", hash = "sha256:90deee9be342a270d07d95049e525c609f9e84319e6f9d1506ae19aa2973f8d5"},
+    {file = "inline_snapshot-0.9.0.tar.gz", hash = "sha256:5f1c4f0fbf7bcbc0d05dc43822032ed4b954d3f20b01868bde7feda8d7c38817"},
 ]
 
 [package.dependencies]
@@ -1106,13 +1106,13 @@ types-toml = ">=0.10.8.7"
 
 [[package]]
 name = "jinja2"
-version = "3.1.3"
+version = "3.1.4"
 description = "A very fast and expressive template engine."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"},
-    {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"},
+    {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"},
+    {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"},
 ]
 
 [package.dependencies]
@@ -1997,13 +1997,13 @@ testing = ["pytest", "pytest-benchmark"]
 
 [[package]]
 name = "pre-commit"
-version = "3.7.0"
+version = "3.7.1"
 description = "A framework for managing and maintaining multi-language pre-commit hooks."
 optional = false
 python-versions = ">=3.9"
 files = [
-    {file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"},
-    {file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"},
+    {file = "pre_commit-3.7.1-py2.py3-none-any.whl", hash = "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5"},
+    {file = "pre_commit-3.7.1.tar.gz", hash = "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a"},
 ]
 
 [package.dependencies]
@@ -2337,13 +2337,13 @@ files = [
 
 [[package]]
 name = "requests"
-version = "2.31.0"
+version = "2.32.0"
 description = "Python HTTP for Humans."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
-    {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
+    {file = "requests-2.32.0-py3-none-any.whl", hash = "sha256:f2c3881dddb70d056c5bd7600a4fae312b2a300e39be6a118d30b90bd27262b5"},
+    {file = "requests-2.32.0.tar.gz", hash = "sha256:fa5490319474c82ef1d2c9bc459d3652e3ae4ef4c4ebdd18a21145a47ca4b6b8"},
 ]
 
 [package.dependencies]
@@ -2394,19 +2394,18 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
 
 [[package]]
 name = "setuptools"
-version = "69.5.1"
+version = "70.0.0"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"},
-    {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"},
+    {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"},
+    {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"},
 ]
 
 [package.extras]
-docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
-testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
-testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
+docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
 
 [[package]]
 name = "shellingham"
@@ -2525,13 +2524,13 @@ files = [
 
 [[package]]
 name = "tqdm"
-version = "4.66.2"
+version = "4.66.3"
 description = "Fast, Extensible Progress Meter"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"},
-    {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"},
+    {file = "tqdm-4.66.3-py3-none-any.whl", hash = "sha256:4f41d54107ff9a223dca80b53efe4fb654c67efaba7f47bada3ee9d50e05bd53"},
+    {file = "tqdm-4.66.3.tar.gz", hash = "sha256:23097a41eba115ba99ecae40d06444c15d1c0c698d527a01c6c8bd1c5d0647e5"},
 ]
 
 [package.dependencies]
@@ -3178,20 +3177,20 @@ multidict = ">=4.0"
 
 [[package]]
 name = "zipp"
-version = "3.18.1"
+version = "3.19.1"
 description = "Backport of pathlib-compatible object wrapper for zip files"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"},
-    {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"},
+    {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"},
+    {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"},
 ]
 
 [package.extras]
-docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
-testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"]
+doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
+test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"]
 
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "b29700a8e3d5d5086bb0acde90e9480d87de4ee0874b85d5a37a635977237525"
+content-hash = "28569362ae4d469cbb095d6d221a0c765ccdc45d6d024f5741a663e6ca5f012c"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agentic_security"
-version = "0.1.4"
+version = "0.1.7"
 description = "Agentic LLM vulnerability scanner"
 authors = ["Alexander Miasoiedov <msoedov@gmail.com>"]
 maintainers = ["Alexander Miasoiedov <msoedov@gmail.com>"]
@@ -28,7 +28,7 @@ agentic_security = "agentic_security.__main__:entrypoint"
 python = "^3.9"
 fastapi = ">=0.109.1,<0.112.0"
 uvicorn = ">=0.23.2,<0.30.0"
-fire = "^0.5.0"
+fire = ">=0.5,<0.7"
 loguru = "^0.7.2"
 httpx = ">=0.25.1,<0.28.0"
 cache-to-disk = "^2.0.0"
@@ -44,7 +44,7 @@ mypy = "^1.6.1"
 httpx = ">=0.25.1,<0.28.0"
 pytest = ">=7.4.3,<9.0.0"
 pre-commit = "^3.5.0"
-inline-snapshot = "^0.8.0"
+inline-snapshot = ">=0.8,<0.10"
 langchain-groq = "^0.1.3"
 
 [tool.ruff]

test.http

@@ -14,7 +14,7 @@ Content-Type: application/json
 
 
 ###
-POST http://0.0.0.0:3008/v1/self-probe
+POST http://0.0.0.0:8718/v1/self-probe
 Authorization: Bearer XXXXX
 Content-Type: application/json