mirror of
https://github.com/msoedov/agentic_security.git
synced 2026-06-24 22:29:56 +02:00
Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 879d2bf819 | |||
| 0a07fc54d6 | |||
| 2f1151d44d | |||
| d0353e3ab9 | |||
| 926c583a17 | |||
| 17e34356e1 | |||
| 312fa756a5 | |||
| 145e7f81e1 | |||
| 04af7d24a1 | |||
| c5c5ae2e4b | |||
| 2bc0605a1d | |||
| 335787d40e | |||
| 1b211b5d76 | |||
| 444f908009 | |||
| f81dc508f9 | |||
| 4a55b99d70 | |||
| 5c2f9eba71 | |||
| aa2fe4d1ad | |||
| cf7c017621 | |||
| 73184e3454 | |||
| 3720ece2af | |||
| 0dc738a11e | |||
| 47ca656d59 | |||
| 4fa166298d |
@@ -19,6 +19,10 @@ RUN poetry lock
|
||||
|
||||
# Install dependencies
|
||||
RUN poetry export -f requirements.txt --without-hashes -o requirements.txt
|
||||
|
||||
# Install wheel (required to build packages like fire)
|
||||
RUN pip install --upgrade pip setuptools wheel
|
||||
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Runtime stage
|
||||
|
||||
@@ -21,9 +21,7 @@
|
||||
<a href="https://pypi.org/project/agentic-security/">
|
||||
<img alt="PyPI Version" src="https://img.shields.io/pypi/v/agentic-security?style=for-the-badge&logo=pypi&labelColor=000000&color=00CCFF" />
|
||||
</a>
|
||||
<a href="https://discord.gg/stw3DfZQ">
|
||||
<img alt="Join Discord" src="https://img.shields.io/badge/Discord-Join%20Us-black?style=for-the-badge&logo=discord&labelColor=000000&color=DD55FF" />
|
||||
</a>
|
||||
|
||||
</p>
|
||||
|
||||
|
||||
|
||||
@@ -11,13 +11,13 @@ server_params = StdioServerParameters(
|
||||
)
|
||||
|
||||
|
||||
async def run():
|
||||
async def run() -> None:
|
||||
async with stdio_client(server_params) as (read, write):
|
||||
async with ClientSession(read, write) as session:
|
||||
# Initialize the connection
|
||||
# Initialize the connection --> connection does not work
|
||||
await session.initialize()
|
||||
|
||||
# List available prompts, resources, and tools
|
||||
# List available prompts, resources, and tools --> no avalialbe tools
|
||||
prompts = await session.list_prompts()
|
||||
print(f"Available prompts: {prompts}")
|
||||
|
||||
@@ -27,7 +27,7 @@ async def run():
|
||||
tools = await session.list_tools()
|
||||
print(f"Available tools: {tools}")
|
||||
|
||||
# Call the echo tool
|
||||
# Call the echo tool --> echo tool iisue
|
||||
echo_result = await session.call_tool(
|
||||
"echo_tool", arguments={"message": "Hello from client!"}
|
||||
)
|
||||
@@ -47,6 +47,7 @@ async def run():
|
||||
# print(f"Prompt result: {prompt_result}")
|
||||
|
||||
# You can perform additional operations here as needed
|
||||
return prompts, resources, tools
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -14,7 +14,15 @@ AGENTIC_SECURITY = "http://0.0.0.0:8718"
|
||||
|
||||
@mcp.tool()
|
||||
async def verify_llm(spec: str) -> dict:
|
||||
"""Verify an LLM model specification using the FastAPI server."""
|
||||
"""
|
||||
Verify an LLM model specification using the FastAPI server
|
||||
|
||||
Returns:
|
||||
dict: containing the verification result form the FastAPI server
|
||||
|
||||
Args: spect(str): The specification of the LLM model to verify.
|
||||
|
||||
"""
|
||||
url = f"{AGENTIC_SECURITY}/verify"
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(url, json={"spec": spec})
|
||||
@@ -28,7 +36,18 @@ async def start_scan(
|
||||
optimize: bool = False,
|
||||
enableMultiStepAttack: bool = False,
|
||||
) -> dict:
|
||||
"""Start an LLM security scan via the FastAPI server."""
|
||||
"""
|
||||
Start an LLM security scan via the FastAPI server.
|
||||
Returns:
|
||||
dict: The scan initiation result from the FastAPI server.
|
||||
|
||||
Args:
|
||||
llmSpec (str): The specification of the LLM model.
|
||||
maxBudget (int): The maximum budget for the scan.
|
||||
optimize (bool, optional): Whether to enable optimization during scanning. Defaults to False.
|
||||
enableMultiStepAttack (bool, optional): Whether to enable multi-step attack
|
||||
|
||||
"""
|
||||
url = f"{AGENTIC_SECURITY}/scan"
|
||||
payload = {
|
||||
"llmSpec": llmSpec,
|
||||
@@ -46,7 +65,11 @@ async def start_scan(
|
||||
|
||||
@mcp.tool()
|
||||
async def stop_scan() -> dict:
|
||||
"""Stop an ongoing scan via the FastAPI server."""
|
||||
"""Stop an ongoing scan via the FastAPI server.
|
||||
|
||||
Returns:
|
||||
dict: The confirmation from the FastAPI server that the scan has been stopped.
|
||||
"""
|
||||
url = f"{AGENTIC_SECURITY}/stop"
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(url)
|
||||
@@ -55,7 +78,12 @@ async def stop_scan() -> dict:
|
||||
|
||||
@mcp.tool()
|
||||
async def get_data_config() -> list:
|
||||
"""Retrieve data configuration from the FastAPI server."""
|
||||
"""
|
||||
Retrieve data configuration from the FastAPI server.
|
||||
|
||||
Returns:
|
||||
list: The response from the FastAPI server, confirming the scan has been stopped.
|
||||
"""
|
||||
url = f"{AGENTIC_SECURITY}/v1/data-config"
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url)
|
||||
@@ -64,7 +92,12 @@ async def get_data_config() -> list:
|
||||
|
||||
@mcp.tool()
|
||||
async def get_spec_templates() -> list:
|
||||
"""Retrieve data configuration from the FastAPI server."""
|
||||
"""
|
||||
Retrieve data configuration from the FastAPI server.
|
||||
|
||||
Returns:
|
||||
list: The LLM specification templates from the FastAPI server.
|
||||
"""
|
||||
url = f"{AGENTIC_SECURITY}/v1/llm-specs"
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(url)
|
||||
|
||||
@@ -245,61 +245,47 @@ def load_jailbreak_v28k() -> ProbeDataset:
|
||||
return create_probe_dataset("JailbreakV-28K/JailBreakV-28k", [])
|
||||
|
||||
|
||||
@cache_to_disk()
|
||||
def load_local_csv() -> ProbeDataset:
|
||||
"""Load prompts from local CSV files."""
|
||||
csv_files = [f for f in os.listdir(".") if f.endswith(".csv")]
|
||||
logger.info(f"Found {len(csv_files)} CSV files: {csv_files}")
|
||||
|
||||
prompts = []
|
||||
for file in csv_files:
|
||||
try:
|
||||
df = pd.read_csv(file)
|
||||
if "prompt" in df.columns:
|
||||
prompts.extend(df["prompt"].tolist())
|
||||
else:
|
||||
logger.warning(f"File {file} lacks a suitable prompt column")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading {file}: {e}")
|
||||
|
||||
return create_probe_dataset("Local CSV", prompts, {"src": str(csv_files)})
|
||||
|
||||
|
||||
@cache_to_disk(1)
|
||||
def load_csv(file: str) -> ProbeDataset:
|
||||
"""Load prompts from local CSV files."""
|
||||
def file_dataset(file) -> list[str]:
|
||||
prompts = []
|
||||
try:
|
||||
df = pd.read_csv(file)
|
||||
prompts = df["prompt"].tolist()
|
||||
df = pd.read_csv(os.path.join("./datasets", file), encoding_errors="ignore")
|
||||
if "prompt" in df.columns:
|
||||
prompts.extend(df["prompt"].tolist())
|
||||
prompts = df["prompt"].tolist()
|
||||
else:
|
||||
logger.warning(f"File {file} lacks a suitable prompt column")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading {file}: {e}")
|
||||
return prompts
|
||||
|
||||
|
||||
def load_local_csv() -> ProbeDataset:
|
||||
"""Load prompts from local CSV files."""
|
||||
os.makedirs("./datasets", exist_ok=True)
|
||||
csv_files = [f for f in os.listdir("./datasets") if f.endswith(".csv")]
|
||||
logger.info(f"Found {len(csv_files)} CSV files: {csv_files}")
|
||||
|
||||
prompts = []
|
||||
for file in csv_files:
|
||||
prompts.extend(file_dataset(file))
|
||||
return create_probe_dataset("Local CSV", prompts, {"src": str(csv_files)})
|
||||
|
||||
|
||||
def load_csv(file: str) -> ProbeDataset:
|
||||
"""Load prompts from local CSV files."""
|
||||
prompts = file_dataset(file)
|
||||
return create_probe_dataset(f"fs://{file}", prompts, {"src": str(file)})
|
||||
|
||||
|
||||
@cache_to_disk(1)
|
||||
def load_local_csv_files() -> list[ProbeDataset]:
|
||||
"""Load prompts from local CSV files and return a list of ProbeDataset objects."""
|
||||
csv_files = [f for f in os.listdir(".") if f.endswith(".csv")]
|
||||
csv_files = [f for f in os.listdir("./datasets") if f.endswith(".csv")]
|
||||
logger.info(f"Found {len(csv_files)} CSV files: {csv_files}")
|
||||
|
||||
datasets = []
|
||||
|
||||
for file in csv_files:
|
||||
try:
|
||||
df = pd.read_csv(file)
|
||||
if "prompt" in df.columns:
|
||||
prompts = df["prompt"].tolist()
|
||||
datasets.append(create_probe_dataset(file, prompts, {"src": file}))
|
||||
else:
|
||||
logger.warning(f"File {file} lacks a suitable prompt column")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading {file}: {e}")
|
||||
|
||||
datasets.append(create_probe_dataset(file, file_dataset(file), {"src": file}))
|
||||
return datasets
|
||||
|
||||
|
||||
@@ -336,12 +322,20 @@ class StenographyTransformer:
|
||||
)
|
||||
continue
|
||||
|
||||
size = len(dataset.prompts)
|
||||
# Convert async generator (if any) to a list
|
||||
prompts = (
|
||||
list(dataset.prompts) if hasattr(dataset.prompts, "__iter__") else []
|
||||
)
|
||||
size = len(prompts)
|
||||
|
||||
if size == 0:
|
||||
continue
|
||||
|
||||
sample_size = max(1, int(size * 0.25)) # At least 1 sample
|
||||
|
||||
for name, fn in self.TRANSFORMATIONS.items():
|
||||
logger.info(f"Applying {name} to {dataset.dataset_name}")
|
||||
sampled = random.sample(dataset.prompts, min(sample_size, size))
|
||||
sampled = random.sample(prompts, min(sample_size, size))
|
||||
try:
|
||||
transformed = [
|
||||
f"Follow the instructions in {name}: {fn(str(p))}"
|
||||
|
||||
Generated
+307
-45
File diff suppressed because it is too large
Load Diff
+2
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "agentic_security"
|
||||
version = "0.7.1"
|
||||
version = "0.7.4"
|
||||
description = "Agentic LLM vulnerability scanner"
|
||||
authors = ["Alexander Miasoiedov <msoedov@gmail.com>"]
|
||||
maintainers = ["Alexander Miasoiedov <msoedov@gmail.com>"]
|
||||
@@ -52,6 +52,7 @@ sentry_sdk = "^2.22.0"
|
||||
orjson = "^3.10"
|
||||
pyfiglet = "^1.0.2"
|
||||
termcolor = "^2.4.0"
|
||||
mcp = "^1.4.1"
|
||||
|
||||
# garak = { version = "*", optional = true }
|
||||
pytest-xdist = "3.6.1"
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import pytest
|
||||
|
||||
from agentic_security.mcp.client import run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_echo_tool():
|
||||
"""Test the echo tool functionality"""
|
||||
prompts, resources, tools = await run()
|
||||
assert prompts
|
||||
assert resources
|
||||
assert tools
|
||||
Generated
+3
-3
@@ -6891,9 +6891,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/http-proxy-middleware": {
|
||||
"version": "2.0.7",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz",
|
||||
"integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==",
|
||||
"version": "2.0.9",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz",
|
||||
"integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
Reference in New Issue
Block a user