mirror of
https://github.com/Shiva108/ai-llm-red-team-handbook.git
synced 2026-02-12 14:42:46 +00:00
docs(lab-setup): overhaul lab setup and safety chapter with practical guides
- Significantly expanded Chapter 7 with detailed guides and code examples for AI red teaming lab setup. - Introduced comprehensive sections on local LLM deployment, API-based testing, and network isolation. - Added critical safety controls including kill switches, watchdog timers, rate limiting, and cost management. - Included advanced topics such as testing RAG, agent systems, and multi-modal models. - Provided pre-engagement and daily operational checklists, risk management, and incident response procedures.
This commit is contained in:
shiva108
95
audit_report.json
Normal file
95
audit_report.json
Normal file
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"metadata": {
|
||||
"chapter": "Chapter_07_Lab_Setup_and_Environmental_Safety.md",
|
||||
"audit_date": "2026-02-03 13:04:24",
|
||||
"duration_seconds": 1.08,
|
||||
"version": "2.0"
|
||||
},
|
||||
"summary": {
|
||||
"total_links": 14,
|
||||
"broken_links": 7
|
||||
},
|
||||
"url_verification": [
|
||||
{
|
||||
"url": "http://localhost:11434/",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:11434",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:8000/v1",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /v1 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0f00f04690>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:7860",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='localhost', port=7860): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0f00eff390>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:8080",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0f00f05fd0>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
},
|
||||
{
|
||||
"url": "http://elasticsearch:9200",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='elasticsearch', port=9200): Max retries exceeded with url: / (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x7f0f1217a5d0>: Failed to resolve 'elasticsearch' ([Errno -2] Name or service not known)\"))"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:11434/api/generate",
|
||||
"status": 405,
|
||||
"alive": false
|
||||
},
|
||||
{
|
||||
"url": "http://ollama:11434",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='ollama', port=11434): Max retries exceeded with url: / (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x7f0f00f07250>: Failed to resolve 'ollama' ([Errno -2] Name or service not known)\"))"
|
||||
},
|
||||
{
|
||||
"url": "https://download.pytorch.org/whl/cpu",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://ollama.com/install.sh",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://api.openai.com/v1/organization/api_keys/${OPENAI_TEMP_KEY_ID}",
|
||||
"status": 404,
|
||||
"alive": false
|
||||
},
|
||||
{
|
||||
"url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/oobabooga/text-generation-webui",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/ggerganov/llama.cpp",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
}
|
||||
],
|
||||
"structure_issues": [
|
||||
{
|
||||
"type": "Structure",
|
||||
"issue": "Duplicate headers found"
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
95
final_audit_report.json
Normal file
95
final_audit_report.json
Normal file
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"metadata": {
|
||||
"chapter": "Chapter_07_Lab_Setup_and_Environmental_Safety.md",
|
||||
"audit_date": "2026-02-03 13:05:24",
|
||||
"duration_seconds": 1.69,
|
||||
"version": "2.0"
|
||||
},
|
||||
"summary": {
|
||||
"total_links": 14,
|
||||
"broken_links": 7
|
||||
},
|
||||
"url_verification": [
|
||||
{
|
||||
"url": "http://localhost:11434",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:8080",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fc113a64f90>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:11434/",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:8000/v1",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /v1 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fc11008c3d0>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:7860",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='localhost', port=7860): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fc1100831d0>: Failed to establish a new connection: [Errno 111] Connection refused'))"
|
||||
},
|
||||
{
|
||||
"url": "http://elasticsearch:9200",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='elasticsearch', port=9200): Max retries exceeded with url: / (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x7fc113a67b50>: Failed to resolve 'elasticsearch' ([Errno -2] Name or service not known)\"))"
|
||||
},
|
||||
{
|
||||
"url": "http://ollama:11434",
|
||||
"status": -1,
|
||||
"alive": false,
|
||||
"error": "HTTPConnectionPool(host='ollama', port=11434): Max retries exceeded with url: / (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x7fc11007e450>: Failed to resolve 'ollama' ([Errno -2] Name or service not known)\"))"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:11434/api/generate",
|
||||
"status": 405,
|
||||
"alive": false
|
||||
},
|
||||
{
|
||||
"url": "https://ollama.com/install.sh",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://download.pytorch.org/whl/cpu",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://api.openai.com/v1/organization/api_keys/${OPENAI_TEMP_KEY_ID}",
|
||||
"status": 404,
|
||||
"alive": false
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/oobabooga/text-generation-webui",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/ggerganov/llama.cpp",
|
||||
"status": 200,
|
||||
"alive": true
|
||||
}
|
||||
],
|
||||
"structure_issues": [
|
||||
{
|
||||
"type": "Structure",
|
||||
"issue": "Duplicate headers found"
|
||||
}
|
||||
]
|
||||
}
|
||||
127
scripts/audit_helper.py
Normal file
127
scripts/audit_helper.py
Normal file
@@ -0,0 +1,127 @@
|
||||
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import argparse
|
||||
import requests
|
||||
import concurrent.futures
|
||||
from urllib.parse import urlparse
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
|
||||
def extract_links(markdown_content: str) -> List[str]:
|
||||
"""Extract all markdown links [text](url) and <url>."""
|
||||
# Match [text](url)
|
||||
md_links = re.findall(r'\[.*?\]\((https?://[^)]+)\)', markdown_content)
|
||||
# Match <url>
|
||||
angle_links = re.findall(r'<(https?://[^>]+)>', markdown_content)
|
||||
# Match bare URLs (ignoring those already captured in () or <>)
|
||||
# This regex looks for http/https, but tries to avoid trailing punctuation often found in text
|
||||
plain_links = re.findall(r'https?://[^\s<>")]+', markdown_content)
|
||||
|
||||
all_links = sorted(list(set(md_links + angle_links + plain_links)))
|
||||
|
||||
# Remove any trailing punctuation that might have slipped in (like periods or commas)
|
||||
clean_links = []
|
||||
for link in all_links:
|
||||
link = link.rstrip('.,;:')
|
||||
clean_links.append(link)
|
||||
|
||||
return sorted(list(set(clean_links)))
|
||||
|
||||
def check_url(url: str, timeout: int = 10) -> Dict[str, Any]:
|
||||
"""Check a single URL for reachability."""
|
||||
try:
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
}
|
||||
response = requests.head(url, headers=headers, timeout=timeout, allow_redirects=True)
|
||||
if response.status_code == 405: # Method Not Allowed, try GET
|
||||
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
||||
response.close()
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"status": response.status_code,
|
||||
"alive": 200 <= response.status_code < 400
|
||||
}
|
||||
except requests.RequestException as e:
|
||||
return {
|
||||
"url": url,
|
||||
"status": -1,
|
||||
"alive": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
def audit_chapter(file_path: str, output_file: str = None) -> Dict[str, Any]:
|
||||
"""Audit a markdown chapter file."""
|
||||
start_time = time.time()
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
print(f"Error: File {file_path} not found.")
|
||||
sys.exit(1)
|
||||
|
||||
content = path.read_text(encoding='utf-8')
|
||||
|
||||
# 1. Verification: Link Checking
|
||||
links = extract_links(content)
|
||||
print(f"Found {len(links)} unique remote URLs. Verifying...")
|
||||
|
||||
url_results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
future_to_url = {executor.submit(check_url, url): url for url in links}
|
||||
for future in concurrent.futures.as_completed(future_to_url):
|
||||
url_results.append(future.result())
|
||||
|
||||
broken_links = [r for r in url_results if not r['alive']]
|
||||
|
||||
# 2. Fact/Formatting Check (Basic Regex)
|
||||
issues = []
|
||||
|
||||
# Check for empty links
|
||||
if re.search(r'\[.*?\]\(\s*\)', content):
|
||||
issues.append({"type": "Formatting", "issue": "Found empty markdown link ()"})
|
||||
|
||||
# Check for duplicate headers
|
||||
headers = re.findall(r'^#+\s+(.+)$', content, re.MULTILINE)
|
||||
if len(headers) != len(set(headers)):
|
||||
issues.append({"type": "Structure", "issue": "Duplicate headers found"})
|
||||
|
||||
total_duration = time.time() - start_time
|
||||
|
||||
report = {
|
||||
"metadata": {
|
||||
"chapter": path.name,
|
||||
"audit_date": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"duration_seconds": round(total_duration, 2),
|
||||
"version": "2.0"
|
||||
},
|
||||
"summary": {
|
||||
"total_links": len(links),
|
||||
"broken_links": len(broken_links)
|
||||
},
|
||||
"url_verification": convert_results_to_json_friendly(url_results),
|
||||
"structure_issues": issues
|
||||
}
|
||||
|
||||
if output_file:
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, indent=2)
|
||||
print(f"Report saved to {output_file}")
|
||||
else:
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
return report
|
||||
|
||||
def convert_results_to_json_friendly(results):
|
||||
# Ensure all data in results is JSON serializable if needed
|
||||
return results
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Audit MD Chapter")
|
||||
parser.add_argument("filename", help="Markdown file to audit")
|
||||
parser.add_argument("--output", help="JSON output file", default="audit_report.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
audit_chapter(args.filename, args.output)
|
||||
Reference in New Issue
Block a user