cleaned and fixed unused stuff

requirements.txt

@@ -1,212 +1,41 @@
-# HexStrike AI MCP Agents v6.0 - Python Dependencies
+# HexStrike AI MCP Agents v6.0 
-# Enhanced with 150+ Security Tools Integration, AI-Powered Intelligence & Browser Agent
-#
-# Instructions if you face libxml issues:
-# 
-# 1. Install system dependencies first (optional, for packages that need compilation):
-#    sudo apt update
-#    sudo apt install python3-dev build-essential
-#
-# 2. Then install Python packages:
-#    pip3 install -r requirements.txt --break-system-packages
 #
+# INSTALLATION COMMANDS:
+# python3 -m venv hexstrike_env
+# source hexstrike_env/bin/activate
+# python3 -m pip install -r requirements.txt
+# python3 hexstrike_server.py
 
 # ============================================================================
-# CORE FRAMEWORK DEPENDENCIES
+# CORE FRAMEWORK DEPENDENCIES (ACTUALLY USED)
 # ============================================================================
-flask>=2.3.0,<4.0.0             # Web framework for API server
+flask>=2.3.0,<4.0.0             # Web framework for API server (flask import)
-requests>=2.31.0,<3.0.0         # HTTP library for API calls and tool integration
+requests>=2.31.0,<3.0.0         # HTTP library (requests import)
-psutil>=5.9.0,<6.0.0            # System and process utilities for monitoring
+psutil>=5.9.0,<6.0.0            # System utilities (psutil import)
-fastmcp>=0.2.0,<1.0.0           # Model Context Protocol framework for AI agents
+fastmcp>=0.2.0,<1.0.0           # MCP framework (from mcp.server.fastmcp import FastMCP)
 
 # ============================================================================
-# DATA PROCESSING & ANALYSIS
+# WEB SCRAPING & AUTOMATION (ACTUALLY USED)
 # ============================================================================
-pandas>=2.0.0,<3.0.0            # Data manipulation and analysis
+beautifulsoup4>=4.12.0,<5.0.0   # HTML parsing (from bs4 import BeautifulSoup)
-numpy>=1.26.4,<2.0.0            # Numerical computing and array operations (Python 3.12+ & 3.13+ compatible)
+selenium>=4.15.0,<5.0.0         # Browser automation (selenium imports)
-python-dateutil>=2.8.0,<3.0.0   # Date and time utilities
+webdriver-manager>=4.0.0,<5.0.0 # ChromeDriver management (referenced in code)
-scipy>=1.11.0,<2.0.0            # Scientific computing and statistical analysis
 
 # ============================================================================
-# NETWORKING & HTTP ENHANCED
+# ASYNC & NETWORKING (ACTUALLY USED)
 # ============================================================================
-urllib3>=2.0.0,<3.0.0           # HTTP client library with connection pooling
+aiohttp>=3.8.0,<4.0.0           # Async HTTP (aiohttp import)
-certifi>=2023.7.0,<2024.0.0     # Certificate authority bundle
-charset-normalizer>=3.2.0,<4.0.0 # Character encoding detection
-httpx>=0.24.0,<1.0.0            # Modern HTTP client for async operations
-aiohttp>=3.8.0,<4.0.0           # Async HTTP client/server framework
-websockets>=11.0.0,<13.0.0      # WebSocket client and server implementation
 
 # ============================================================================
-# JSON & DATA SERIALIZATION
+# PROXY & TESTING (ACTUALLY USED)
 # ============================================================================
-jsonschema>=4.19.0,<5.0.0       # JSON schema validation
+mitmproxy>=9.0.0,<11.0.0        # HTTP proxy (mitmproxy imports)
-pydantic>=2.3.0,<3.0.0          # Data validation using Python type annotations
-orjson>=3.9.0,<4.0.0            # Fast JSON serialization library
-msgpack>=1.0.0,<2.0.0           # Binary serialization format
 
 # ============================================================================
-# LOGGING & MONITORING ENHANCED
+# BINARY ANALYSIS (CONDITIONALLY USED)
 # ============================================================================
-colorama>=0.4.0,<1.0.0          # Cross-platform colored terminal text
+pwntools>=4.10.0,<5.0.0         # Binary exploitation (from pwn import *)
-rich>=13.5.0,<14.0.0            # Rich text and beautiful formatting
+angr>=9.2.0,<10.0.0             # Binary analysis (import angr)
-tqdm>=4.66.0,<5.0.0             # Progress bars for long-running operations
-loguru>=0.7.0,<1.0.0            # Enhanced logging with better formatting
-structlog>=23.1.0,<24.0.0       # Structured logging for better analysis
-
-# ============================================================================
-# SECURITY & CRYPTOGRAPHY ENHANCED
-# ============================================================================
-cryptography>=41.0.0,<42.0.0    # Cryptographic recipes and primitives
-pycryptodome>=3.18.0,<4.0.0     # Cryptographic library with additional algorithms
-bcrypt>=4.0.0,<5.0.0            # Password hashing library
-passlib>=1.7.0,<2.0.0           # Password hashing framework
-jwt>=1.3.0,<2.0.0               # JSON Web Token implementation
-pyotp>=2.9.0,<3.0.0             # One-time password library
-
-# ============================================================================
-# FILE PROCESSING & ANALYSIS ENHANCED
-# ============================================================================
-python-magic>=0.4.0,<1.0.0      # File type identification using libmagic
-pillow>=10.0.0,<11.0.0          # Python Imaging Library for image processing
-exifread>=3.0.0,<4.0.0          # EXIF metadata extraction from images
-pdfplumber>=0.9.0,<1.0.0        # PDF text extraction and analysis
-python-docx>=0.8.0,<2.0.0       # Microsoft Word document processing
-openpyxl>=3.1.0,<4.0.0          # Excel file processing
-
-# ============================================================================
-# DATABASE & STORAGE ENHANCED
-# ============================================================================
-# sqlite3 is built into Python 3.8+ - no separate package needed
-sqlalchemy>=2.0.0,<3.0.0        # SQL toolkit and ORM
-redis>=4.6.0,<6.0.0             # Redis client for caching
-pymongo>=4.5.0,<5.0.0           # MongoDB driver for document storage
-
-# ============================================================================
-# THREADING & CONCURRENCY ENHANCED
-# ============================================================================
-asyncio-throttle>=1.0.0,<2.0.0  # Rate limiting for async operations
-aiofiles>=23.2.0,<24.0.0        # Async file operations
-
-# ============================================================================
-# SYSTEM INTEGRATION ENHANCED
-# ============================================================================
-# signal is built into Python - no separate package needed
-click>=8.1.0,<9.0.0             # Command line interface creation
-typer>=0.9.0,<1.0.0             # Modern CLI framework
-pexpect>=4.8.0,<5.0.0           # Expect-like subprocess control
-
-# ============================================================================
-# BROWSER AGENT & WEB AUTOMATION (v6.0 ENHANCED!)
-# ============================================================================
-selenium>=4.15.0,<5.0.0         # Web browser automation for Browser Agent
-webdriver-manager>=4.0.0,<5.0.0 # Automatic ChromeDriver management
-beautifulsoup4>=4.12.0,<5.0.0   # HTML/XML parsing for DOM analysis
-# lxml not needed - BeautifulSoup uses built-in html.parser
-scrapy>=2.11.0,<3.0.0           # Web scraping framework
-playwright>=1.37.0,<2.0.0       # Modern web automation library
-mitmproxy>=9.0.0,<11.0.0        # HTTP proxy for traffic analysis
-pyppeteer>=1.0.0,<2.0.0         # Headless Chrome automation alternative
-
-# ============================================================================
-# NETWORK ANALYSIS & MONITORING
-# ============================================================================
-scapy>=2.5.0,<3.0.0             # Packet manipulation and network analysis
-netaddr>=0.8.0,<1.0.0           # Network address manipulation
-dnspython>=2.4.0,<3.0.0         # DNS toolkit for Python
-python-nmap>=0.7.0,<1.0.0       # Nmap integration library
-pyshark>=0.6.0,<1.0.0           # Wireshark packet analysis
-
-# ============================================================================
-# BINARY ANALYSIS & REVERSE ENGINEERING
-# ============================================================================
-pwntools>=4.10.0,<5.0.0         # CTF framework and exploit development
-capstone>=5.0.0,<6.0.0          # Disassembly framework
-keystone-engine>=0.9.0,<1.0.0   # Assembler framework
-unicorn>=2.0.0,<3.0.0           # CPU emulator framework
-angr>=9.2.0,<10.0.0             # Binary analysis platform
-r2pipe>=1.7.0,<2.0.0            # Radare2 Python bindings
-
-# ============================================================================
-# CLOUD & CONTAINER SECURITY
-# ============================================================================
-boto3>=1.28.0,<2.0.0            # AWS SDK for Python
-azure-identity>=1.14.0,<2.0.0   # Azure authentication library
-azure-mgmt-security>=5.0.0,<6.0.0 # Azure security management
-google-cloud-security-center>=1.23.0,<2.0.0 # Google Cloud Security Center
-kubernetes>=27.0.0,<29.0.0      # Kubernetes Python client
-docker>=6.1.0,<7.0.0            # Docker SDK for Python
-
-# ============================================================================
-# AI & MACHINE LEARNING (OPTIONAL - Install separately if needed)
-# ============================================================================
-# These packages are NOT used in the core codebase and are OPTIONAL
-# Install separately if you want to add custom AI features:
-#   pip3 install torch transformers sentence-transformers
-#   pip3 install openai anthropic langchain tiktoken scikit-learn
-# See README.md for detailed AI setup instructions
-
-# ============================================================================
-# DEVELOPMENT & TESTING ENHANCED
-# ============================================================================
-pytest>=7.4.0,<8.0.0            # Testing framework
-pytest-asyncio>=0.21.0,<1.0.0   # Async testing support
-pytest-cov>=4.1.0,<5.0.0        # Coverage reporting
-pytest-mock>=3.11.0,<4.0.0      # Mocking utilities
-black>=23.7.0,<24.0.0           # Code formatter
-flake8>=6.0.0,<7.0.0            # Code linting
-mypy>=1.5.0,<2.0.0              # Static type checking
-bandit>=1.7.0,<2.0.0            # Security linting
-safety>=2.3.0,<3.0.0            # Dependency vulnerability scanning
-
-# ============================================================================
-# ADVANCED PROCESS MANAGEMENT & MONITORING (v6.0 ENHANCED!)
-# ============================================================================
-memory-profiler>=0.61.0,<1.0.0  # Memory usage profiling
-py-spy>=0.3.0,<1.0.0            # Sampling profiler for Python
-prometheus-client>=0.17.0,<1.0.0 # Prometheus metrics client
-grafana-api>=1.0.0,<2.0.0       # Grafana API client
-cachetools>=5.3.0,<6.0.0        # Advanced caching utilities
-diskcache>=5.6.0,<6.0.0         # Disk-based caching for large datasets
-joblib>=1.3.0,<2.0.0            # Parallel processing and caching
-
-# ============================================================================
-# VULNERABILITY INTELLIGENCE & CVE ANALYSIS (v6.0 NEW!)
-# ============================================================================
-cve-search-api>=1.0.0,<2.0.0    # CVE database API integration
-nvdlib>=0.7.0,<1.0.0            # NIST NVD API client for CVE data
-exploit-db>=1.3.0,<2.0.0        # Exploit Database integration
-mitre-cve>=1.1.0,<2.0.0         # MITRE CVE database client
-vulners>=2.0.0,<3.0.0           # Vulners API for vulnerability intelligence
-
-# ============================================================================
-# SPECIALIZED SECURITY LIBRARIES ENHANCED
-# ============================================================================
-yara-python>=4.3.0,<5.0.0       # YARA pattern matching engine
-python-whois>=0.8.0,<1.0.0      # WHOIS protocol client
-shodan>=1.29.0,<2.0.0           # Shodan API client
-censys>=2.2.0,<3.0.0            # Censys API client
-virustotal-api>=1.1.0,<2.0.0    # VirusTotal API client
-haveibeenpwned>=3.0.0,<4.0.0    # Have I Been Pwned API client
-threatintel>=1.0.0,<2.0.0       # Threat intelligence correlation
-iocextract>=1.16.0,<2.0.0       # IOC extraction from text
-
-# ============================================================================
-# STEGANOGRAPHY & FORENSICS
-# ============================================================================
-stegano>=0.11.0,<1.0.0          # Steganography tools
-pillow-heif>=0.13.0,<1.0.0      # HEIF image format support
-volatility3>=2.4.0,<3.0.0       # Memory forensics framework
-
-# ============================================================================
-# CRYPTOGRAPHY & HASH ANALYSIS
-# ============================================================================
-hashid>=3.1.0,<4.0.0            # Hash identifier
-
-# ============================================================================
-# OSINT & RECONNAISSANCE
-# ============================================================================
-sublist3r>=1.1.0,<2.0.0         # Subdomain enumeration tool
 
 # ============================================================================
 # EXTERNAL SECURITY TOOLS (150+ Tools - Install separately)
@@ -244,20 +73,11 @@ sublist3r>=1.1.0,<2.0.0         # Subdomain enumeration tool
 # - sherlock, social-analyzer, recon-ng, maltego, spiderfoot
 # - shodan-cli, censys-cli, have-i-been-pwned
 # 
-# 🌐 Browser Agent Requirements:
-# - Chrome/Chromium browser and ChromeDriver for headless automation
-# - Burp Suite integration support for proxy chaining
-# 
-# 🤖 AI Agent Dependencies:
-# - FastMCP framework for AI agent communication (included)
-# - Optional AI/ML packages available separately (see README.md)
-# 
 # Installation Notes:
 # 1. Kali Linux 2024.1+ includes most tools by default
 # 2. Ubuntu/Debian users should install tools from official repositories
 # 3. Some tools require compilation from source or additional setup
 # 4. Cloud tools require API keys and authentication configuration
 # 5. Browser Agent requires Chrome/Chromium and ChromeDriver installation
-# 6. AI features require API keys for OpenAI/Anthropic services (optional)
 # 
 # For complete installation instructions and setup guides, see README.md