feat(add Inspect AI):

agentic_security/probe_data/__init__.py

@@ -141,6 +141,16 @@ REGISTRY = [
         "url": "https://github.com/leondz/garak2",
         "dynamic": True,
     },
+    {
+        "dataset_name": "InspectAI",
+        "num_prompts": 0,
+        "tokens": 0,
+        "approx_cost": 0.0,
+        "source": "Github: https://github.com/UKGovernmentBEIS/inspect_ai",
+        "selected": False,
+        "url": "https://github.com/UKGovernmentBEIS/inspect_ai",
+        "dynamic": True,
+    },
     {
         "dataset_name": "Custom CSV",
         "num_prompts": len(load_local_csv().prompts),

agentic_security/probe_data/data.py

@@ -7,7 +7,11 @@ import pandas as pd
 from loguru import logger
 
 from agentic_security.probe_data import stenography_fn
-from agentic_security.probe_data.modules import adaptive_attacks, garak_tool
+from agentic_security.probe_data.modules import (
+    adaptive_attacks,
+    garak_tool,
+    inspect_ai_tool,
+)
 
 IS_VERCEL = os.getenv("IS_VERCEL", "f") == "t"
 
@@ -206,6 +210,11 @@ def prepare_prompts(dataset_names, budget, tools_inbox=None):
             garak_tool.Module(group, tools_inbox=tools_inbox).apply(),
             lazy=True,
         ),
+        "InspectAI": lambda: dataset_from_iterator(
+            "InspectAI",
+            inspect_ai_tool.Module(group, tools_inbox=tools_inbox).apply(),
+            lazy=True,
+        ),
         "GPT fuzzer": lambda: [],
     }
 

agentic_security/probe_data/modules/garak_tool.py

@@ -9,7 +9,6 @@ from loguru import logger
 
 
 class Module:
-
     def __init__(self, prompt_groups: [], tools_inbox: asyncio.Queue):
         self.tools_inbox = tools_inbox
         if not self.is_garak_installed():

agentic_security/probe_data/modules/inspect_ai_task.py

@@ -0,0 +1,13 @@
+from inspect_ai import Task, eval, task
+from inspect_ai.dataset import example_dataset
+from inspect_ai.scorer import model_graded_fact
+from inspect_ai.solver import chain_of_thought, generate, self_critique
+
+
+@task
+def theory_of_mind():
+    return Task(
+        dataset=example_dataset("theory_of_mind"),
+        plan=[chain_of_thought(), generate(), self_critique()],
+        scorer=model_graded_fact(),
+    )

agentic_security/probe_data/modules/inspect_ai_tool.py

@@ -0,0 +1,71 @@
+import asyncio
+import importlib.util
+import os
+
+from loguru import logger
+
+inspect_ai_task = (
+    __file__.replace("inspect_ai_tool.py", "inspect_ai_task.py")
+    .replace(os.getcwd(), "")
+    .strip("/")
+)
+
+
+class Module:
+    name = "Inspect AI"
+
+    def __init__(self, prompt_groups: [], tools_inbox: asyncio.Queue):
+        self.tools_inbox = tools_inbox
+        if not self.is_tool_installed():
+            logger.error(
+                "inspect_ai module is not installed. Please install it using 'pip install inspect_ai'"
+            )
+
+    def is_tool_installed(self) -> bool:
+        inspect_ai = importlib.util.find_spec("inspect_ai")
+        return inspect_ai is not None
+
+    async def _proc(self, command):
+        env = os.environ.copy()
+        env["OPENAI_API_BASE"] = "http://0.0.0.0:8718/proxy"
+        process = await asyncio.create_subprocess_shell(
+            command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+            shell=True,
+        )
+
+        logger.info(f"Started {command}")
+
+        # Read output as it becomes available
+        async for line in process.stdout:
+            logger.info(line.decode().strip())
+
+        # Check for errors
+        err = await process.stderr.read()
+        if err:
+            logger.error(err.decode().strip())
+
+        await process.wait()
+        logger.info(f"Command {command} {process}finished.")
+
+    async def apply(self) -> []:
+        env = os.environ.copy()
+        env["OPENAI_API_BASE"] = "http://0.0.0.0:8718/proxy"
+
+        # Command to be executed
+        command = f"inspect eval {inspect_ai_task} --model openai/gpt-4  --model-base-url=http://0.0.0.0:8718/proxy"
+        logger.info(f"Executing command: {command}")
+
+        proc = asyncio.create_task(self._proc(command))
+        is_empty = self.tools_inbox.empty()
+        await asyncio.sleep(2)
+        logger.info(f"Is inbox empty? {is_empty}")
+        while not self.tools_inbox.empty():
+            ref = self.tools_inbox.get_nowait()
+            message, _, ready = ref["message"], ref["reply"], ref["ready"]
+            yield message
+            ready.set()
+        logger.info(f"{self.name} tool finished.")
+        await proc