test: Add secret detection benchmark dataset and ground truth

Add comprehensive benchmark dataset with 32 documented secrets for testing
secret detection workflows (gitleaks, trufflehog, llm_secret_detection).

- Add test_projects/secret_detection_benchmark/ with 19 test files
- Add ground truth JSON with precise line-by-line secret mappings
- Update .gitignore with exceptions for benchmark files (not real secrets)

Dataset breakdown:
- 12 Easy secrets (standard patterns)
- 10 Medium secrets (obfuscated)
- 10 Hard secrets (well hidden)
This commit is contained in:
tduhamel42
2025-10-16 11:46:28 +02:00
parent 87e3262832
commit 3be4d34531
22 changed files with 773 additions and 0 deletions
@@ -0,0 +1,25 @@
package main
import (
"fmt"
"strings"
)
// HARD SECRET #29: Heredoc with unusual delimiter
const ConfigTemplate = `
SECRET_KEY=golang_heredoc_secret_999
END_OF_CONFIG
`
// HARD SECRET #30: Secret with intentional typo corrected programmatically
const API_KEY_TYPO = "strippe_sk_live_corrected_key"
func CorrectTypo(s string) string {
return strings.Replace(s, "strippe", "stripe", 1)
}
func main() {
fmt.Println("Crypto utilities initialized")
correctedKey := CorrectTypo(API_KEY_TYPO)
fmt.Println("Key ready:", correctedKey[:10]+"...")
}
@@ -0,0 +1,10 @@
package com.example.benchmark;
public class Main {
// EASY SECRET #10: Google OAuth secret in Java
private static final String GOOGLE_OAUTH_SECRET = "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";
public static void main(String[] args) {
System.out.println("Application starting...");
}
}
@@ -0,0 +1,19 @@
// Advanced obfuscation techniques
// HARD SECRET #25: Template string with escaping
const SECRET_TEMPLATE = `sk_${"prod"}_${"template"}_${"key"}_xyz`;
// HARD SECRET #26: Secret in regex pattern
const PASSWORD_REGEX = /password_regex_secret_789/;
// HARD SECRET #27: XORed secret (XOR with key 42)
const XOR_SECRET = [65,82,90,75,94,91,92,75,93,67,65,90,67,92,75,91,67,95];
function decodeXOR() {
return String.fromCharCode(...XOR_SECRET.map(c => c ^ 42));
}
// HARD SECRET #28: Escaped JSON within string
const CONFIG_JSON = "{\"api_key\":\"sk_escaped_json_key_456\"}";
module.exports = { SECRET_TEMPLATE, decodeXOR };
@@ -0,0 +1,19 @@
"""
Main application entry point
"""
import os
# EASY SECRET #5: JWT Secret
JWT_SECRET_KEY = "my-super-secret-jwt-key-do-not-share-2024"
def init_app():
"""Initialize the application"""
app_config = {
"name": "SecretDetectionBenchmark",
"version": "1.0.0"
}
return app_config
if __name__ == "__main__":
print("Application starting...")
init_app()
@@ -0,0 +1,19 @@
"""
Configuration with moderately obfuscated secrets
"""
import base64
# MEDIUM SECRET #11: Base64 encoded AWS key
AWS_KEY_ENCODED = "QUtJQUlPU0ZPRE5ON0VYQU1QTEU="
# MEDIUM SECRET #12: Hex-encoded API token
HEX_TOKEN = "6170695f746f6b656e5f616263313233787977373839"
# MEDIUM SECRET #13: Split secret concatenated at runtime
DB_PASS_PART1 = "MySecure"
DB_PASS_PART2 = "Password"
DB_PASS_PART3 = "2024!"
DATABASE_PASSWORD = DB_PASS_PART1 + DB_PASS_PART2 + DB_PASS_PART3
def get_aws_key():
return base64.b64decode(AWS_KEY_ENCODED).decode()
@@ -0,0 +1,15 @@
-- Database initialization script
CREATE DATABASE prod_db;
-- MEDIUM SECRET #18: Secret in SQL comment
-- Connection string: postgresql://admin:Pr0dDB_S3cr3t_P@ss@db.prod.example.com:5432/prod_db
CREATE TABLE users (
id SERIAL PRIMARY KEY,
username VARCHAR(255) NOT NULL,
email VARCHAR(255) NOT NULL
);
-- Insert test data
INSERT INTO users (username, email) VALUES ('admin', 'admin@example.com');
@@ -0,0 +1,23 @@
"""
Heavily obfuscated secrets - hard to detect
"""
import codecs
# HARD SECRET #21: ROT13 encoded secret
SECRET_ROT13 = "fx_yvir_frperg_xrl_12345"
# HARD SECRET #22: Binary string representation
GITHUB_TOKEN_BYTES = b'\x67\x68\x70\x5f\x4d\x79\x47\x69\x74\x48\x75\x62\x54\x6f\x6b\x65\x6e\x31\x32\x33\x34\x35\x36'
# HARD SECRET #23: Character array join
AWS_SECRET_CHARS = ['A','W','S','_','S','E','C','R','E','T','_','K','E','Y','_','X','Y','Z','7','8','9']
AWS_SECRET = ''.join(AWS_SECRET_CHARS)
# HARD SECRET #24: Reversed string that's un-reversed at runtime
TOKEN_REVERSED = "321cba_desrever_nekot_ipa"
def get_rot13_secret():
return codecs.decode(SECRET_ROT13, 'rot_13')
def get_token():
return TOKEN_REVERSED[::-1]