agentic_security/prd.json

{
  "title": "Agentic Security Enhancements from Research",
  "description": "Integrate features and approaches from promptfoo, promptmap, and FuzzyAI research directories to improve the LLM pentesting capabilities",
  "branchName": "feat/research-enhancements",
  "userStories": [
    {
      "id": "US-001",
      "title": "Dual-LLM Evaluation for Attack Success Detection",
      "description": "Integrate a controller LLM to evaluate attack success instead of relying solely on marker-based refusal detection. Inspired by Promptmap's dual-LLM architecture where a separate LLM judges if the target LLM was successfully attacked.",
      "acceptanceCriteria": [
        "Create LLMRefusalClassifier that uses an LLM to evaluate if a response indicates successful attack",
        "Integrate with existing RefusalClassifierPlugin system",
        "Support configurable evaluation prompts",
        "Add unit tests for the new classifier"
      ],
      "priority": 1,
      "passes": true
    },
    {
      "id": "US-002",
      "title": "YAML-based Attack Rule System",
      "description": "Create a YAML-based rule system for defining attack patterns and success conditions. Inspired by Promptmap's 50+ YAML rule definitions that externalize attack logic from code.",
      "acceptanceCriteria": [
        "Define YAML schema for attack rules with prompt templates and success conditions",
        "Create rule loader that parses YAML files into attack configurations",
        "Support custom user-defined rules",
        "Add unit tests for rule loading and validation"
      ],
      "priority": 2,
      "passes": true
    },
    {
      "id": "US-003",
      "title": "Composable Fuzzing Chain System",
      "description": "Implement a composable chain system for multi-step attacks using pipe operator syntax. Inspired by FuzzyAI's FuzzNode/FuzzChain architecture that allows chaining LLM calls.",
      "acceptanceCriteria": [
        "Create FuzzNode class for individual attack steps with template variable support",
        "Create FuzzChain class that composes nodes using pipe operator",
        "Support template variable substitution between chain steps",
        "Add unit tests for chain composition and execution"
      ],
      "priority": 3,
      "passes": false
    },
    {
      "id": "US-004",
      "title": "Unified LLM Provider Abstraction",
      "description": "Create a unified provider abstraction layer for direct LLM integrations beyond HTTP specs. Inspired by FuzzyAI's comprehensive provider system supporting OpenAI, Anthropic, Gemini, etc.",
      "acceptanceCriteria": [
        "Create BaseLLMProvider abstract class with standard interface",
        "Implement OpenAI and Anthropic provider classes",
        "Create provider factory for instantiation by name",
        "Add unit tests for provider implementations"
      ],
      "priority": 4,
      "passes": false
    },
    {
      "id": "US-005",
      "title": "Enhanced Refusal Detection with Hybrid Approach",
      "description": "Combine marker-based detection with statistical and LLM-based detection for more accurate refusal classification. Enhance the existing refusal detection to reduce false positives/negatives.",
      "acceptanceCriteria": [
        "Add confidence scoring to refusal detection",
        "Implement hybrid classifier that combines multiple detection methods",
        "Support configurable detection thresholds",
        "Add unit tests for hybrid detection"
      ],
      "priority": 5,
      "passes": false
    }
  ]
}