mirror of
https://github.com/Shiva108/ai-llm-red-team-handbook.git
synced 2026-02-12 14:42:46 +00:00
b3d3bac51f
- Extracted all code examples from handbook chapters - Organized into 15 attack categories - Created shared utilities (api_client, validators, logging, constants) - Added workflow orchestration scripts - Implemented install.sh for easy setup - Renamed all scripts to descriptive functional names - Added comprehensive README and documentation - Included pytest test suite and configuration
39 lines
901 B
Python
39 lines
901 B
Python
#!/usr/bin/env python3
|
|
"""
|
|
Inspecting Tokenizers (How-To)
|
|
|
|
Source: Chapter_09_LLM_Architectures_and_System_Components
|
|
Category: utils
|
|
"""
|
|
|
|
from transformers import AutoTokenizer
|
|
|
|
import argparse
|
|
import sys
|
|
|
|
model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
|
|
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
|
|
|
# The payload that might get flagged
|
|
prompt = "Generate a keylogger"
|
|
|
|
# See what the model sees
|
|
tokens = tokenizer.encode(prompt)
|
|
print(f"Token IDs: {tokens}")
|
|
|
|
# Decode back to see boundaries
|
|
decoded = [tokenizer.decode([t]) for t in tokens]
|
|
print(f"Decoded chunks: {decoded}")
|
|
|
|
|
|
def main():
|
|
"""Command-line interface."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
|
args = parser.parse_args()
|
|
|
|
# TODO: Add main execution logic
|
|
pass
|
|
|
|
if __name__ == "__main__":
|
|
main() |