From 3be4d345319fafe673b99dc8f896d75ea2acf488 Mon Sep 17 00:00:00 2001 From: tduhamel42 Date: Thu, 16 Oct 2025 11:46:28 +0200 Subject: [PATCH] test: Add secret detection benchmark dataset and ground truth Add comprehensive benchmark dataset with 32 documented secrets for testing secret detection workflows (gitleaks, trufflehog, llm_secret_detection). - Add test_projects/secret_detection_benchmark/ with 19 test files - Add ground truth JSON with precise line-by-line secret mappings - Update .gitignore with exceptions for benchmark files (not real secrets) Dataset breakdown: - 12 Easy secrets (standard patterns) - 10 Medium secrets (obfuscated) - 10 Hard secrets (well hidden) --- .gitignore | 6 + ...cret_detection_benchmark_GROUND_TRUTH.json | 344 ++++++++++++++++++ test_projects/secret_detection_benchmark/.env | 7 + .../.fuzzforge/findings.db | Bin 0 -> 53248 bytes .../secret_detection_benchmark/README.md | 99 +++++ .../config/app.properties | 9 + .../config/database.yaml | 10 + .../config/keys.yaml | 12 + .../config/legacy.ini | 8 + .../config/oauth.json | 11 + .../config/settings.py | 21 ++ .../secret_detection_benchmark/id_rsa | 7 + .../scripts/deploy.sh | 16 + .../scripts/webhook.js | 13 + .../secret_detection_benchmark/src/Crypto.go | 25 ++ .../secret_detection_benchmark/src/Main.java | 10 + .../src/advanced.js | 19 + .../secret_detection_benchmark/src/app.py | 19 + .../secret_detection_benchmark/src/config.py | 19 + .../src/database.sql | 15 + .../src/obfuscated.py | 23 ++ .../validate_ground_truth.py | 80 ++++ 22 files changed, 773 insertions(+) create mode 100644 backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json create mode 100644 test_projects/secret_detection_benchmark/.env create mode 100644 test_projects/secret_detection_benchmark/.fuzzforge/findings.db create mode 100644 test_projects/secret_detection_benchmark/README.md create mode 100644 test_projects/secret_detection_benchmark/config/app.properties create mode 100644 test_projects/secret_detection_benchmark/config/database.yaml create mode 100644 test_projects/secret_detection_benchmark/config/keys.yaml create mode 100644 test_projects/secret_detection_benchmark/config/legacy.ini create mode 100644 test_projects/secret_detection_benchmark/config/oauth.json create mode 100644 test_projects/secret_detection_benchmark/config/settings.py create mode 100644 test_projects/secret_detection_benchmark/id_rsa create mode 100644 test_projects/secret_detection_benchmark/scripts/deploy.sh create mode 100644 test_projects/secret_detection_benchmark/scripts/webhook.js create mode 100644 test_projects/secret_detection_benchmark/src/Crypto.go create mode 100644 test_projects/secret_detection_benchmark/src/Main.java create mode 100644 test_projects/secret_detection_benchmark/src/advanced.js create mode 100644 test_projects/secret_detection_benchmark/src/app.py create mode 100644 test_projects/secret_detection_benchmark/src/config.py create mode 100644 test_projects/secret_detection_benchmark/src/database.sql create mode 100644 test_projects/secret_detection_benchmark/src/obfuscated.py create mode 100644 test_projects/secret_detection_benchmark/validate_ground_truth.py diff --git a/.gitignore b/.gitignore index dd922f9..b090789 100644 --- a/.gitignore +++ b/.gitignore @@ -233,6 +233,12 @@ yarn-error.log* *.key *.p12 *.pfx + +# Exception: Secret detection benchmark test files (not real secrets) +!test_projects/secret_detection_benchmark/ +!test_projects/secret_detection_benchmark/** +!**/secret_detection_benchmark_GROUND_TRUTH.json + secret* secrets/ credentials* diff --git a/backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json b/backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json new file mode 100644 index 0000000..cd6223c --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json @@ -0,0 +1,344 @@ +{ + "description": "Ground truth dataset for secret detection benchmarking - Exactly 32 secrets", + "version": "1.1.0", + "total_secrets": 32, + "secrets_by_difficulty": { + "easy": 12, + "medium": 10, + "hard": 10 + }, + "secrets": [ + { + "id": 1, + "file": ".env", + "line": 3, + "difficulty": "easy", + "type": "aws_access_key", + "value": "AKIAIOSFODNN7EXAMPLE", + "severity": "critical" + }, + { + "id": 2, + "file": ".env", + "line": 4, + "difficulty": "easy", + "type": "aws_secret_access_key", + "value": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "severity": "critical" + }, + { + "id": 3, + "file": "config/settings.py", + "line": 6, + "difficulty": "easy", + "type": "github_pat", + "value": "ghp_vR8jK2mN4pQ6tX9bC3wY7zA1eF5hI8kL", + "severity": "critical" + }, + { + "id": 4, + "file": "config/settings.py", + "line": 9, + "difficulty": "easy", + "type": "stripe_api_key", + "value": "sk_live_51MabcdefghijklmnopqrstuvwxyzABCDEF123456789", + "severity": "critical" + }, + { + "id": 5, + "file": "config/settings.py", + "line": 17, + "difficulty": "easy", + "type": "database_password", + "value": "ProdDB_P@ssw0rd_2024_Secure!", + "severity": "critical" + }, + { + "id": 6, + "file": "src/app.py", + "line": 6, + "difficulty": "easy", + "type": "jwt_secret", + "value": "my-super-secret-jwt-key-do-not-share-2024", + "severity": "critical" + }, + { + "id": 7, + "file": "config/database.yaml", + "line": 7, + "difficulty": "easy", + "type": "azure_storage_key", + "value": "DefaultEndpointsProtocol=https;AccountName=prodstore;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;EndpointSuffix=core.windows.net", + "severity": "critical" + }, + { + "id": 8, + "file": "scripts/webhook.js", + "line": 4, + "difficulty": "easy", + "type": "slack_webhook", + "value": "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXX", + "severity": "high" + }, + { + "id": 9, + "file": "config/app.properties", + "line": 6, + "difficulty": "easy", + "type": "api_key", + "value": "sk_test_4eC39HqLyjWDarjtT1zdp7dc", + "severity": "high" + }, + { + "id": 10, + "file": "id_rsa", + "line": 1, + "difficulty": "easy", + "type": "ssh_private_key", + "value": "-----BEGIN OPENSSH PRIVATE KEY-----", + "severity": "critical" + }, + { + "id": 11, + "file": "config/oauth.json", + "line": 4, + "difficulty": "easy", + "type": "oauth_client_secret", + "value": "GOCSPX-Ab12Cd34Ef56Gh78Ij90Kl12", + "severity": "critical" + }, + { + "id": 12, + "file": "src/Main.java", + "line": 5, + "difficulty": "easy", + "type": "google_oauth_secret", + "value": "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", + "severity": "critical" + }, + { + "id": 13, + "file": "src/config.py", + "line": 7, + "difficulty": "medium", + "type": "aws_access_key_base64", + "value": "QUtJQUlPU0ZPRE5ON0VYQU1QTEU=", + "decoded": "AKIAIOSFODNN7EXAMPLE", + "severity": "critical" + }, + { + "id": 14, + "file": "src/config.py", + "line": 10, + "difficulty": "medium", + "type": "api_token_hex", + "value": "6170695f746f6b656e5f616263313233787977373839", + "decoded": "api_token_abc123xyz789", + "severity": "high" + }, + { + "id": 15, + "file": "src/config.py", + "line": 16, + "difficulty": "medium", + "type": "database_password_concatenated", + "value": "MySecurePassword2024!", + "note": "Built from DB_PASS_PART1 + DB_PASS_PART2 + DB_PASS_PART3", + "severity": "critical" + }, + { + "id": 16, + "file": "scripts/deploy.sh", + "line": 5, + "difficulty": "medium", + "type": "api_key_export", + "value": "sk_prod_1234567890abcdefghijklmnopqrstuvwxyz", + "severity": "critical" + }, + { + "id": 17, + "file": "scripts/deploy.sh", + "line": 11, + "difficulty": "medium", + "type": "database_password_url_encoded", + "value": "mysql://admin:MyP%40ssw0rd%21@db.example.com:3306/prod", + "decoded": "mysql://admin:MyP@ssw0rd!@db.example.com:3306/prod", + "note": "In comment", + "severity": "critical" + }, + { + "id": 18, + "file": "config/keys.yaml", + "line": 6, + "difficulty": "medium", + "type": "rsa_private_key_multiline", + "value": "-----BEGIN RSA PRIVATE KEY-----", + "note": "Multi-line YAML literal block", + "severity": "critical" + }, + { + "id": 19, + "file": "config/keys.yaml", + "line": 11, + "difficulty": "medium", + "type": "api_token_unicode", + "value": "tøkęn_śęçrėt_ẃïth_ŭñïçődė_123456", + "severity": "high" + }, + { + "id": 20, + "file": "src/database.sql", + "line": 6, + "difficulty": "medium", + "type": "database_connection_string", + "value": "postgresql://admin:Pr0dDB_S3cr3t_P@ss@db.prod.example.com:5432/prod_db", + "note": "In SQL comment", + "severity": "critical" + }, + { + "id": 21, + "file": "config/legacy.ini", + "line": 3, + "difficulty": "medium", + "type": "database_password", + "value": "L3g@cy_DB_P@ssw0rd_2023", + "severity": "critical" + }, + { + "id": 22, + "file": "config/legacy.ini", + "line": 7, + "difficulty": "medium", + "type": "api_key_commented", + "value": "backup_key_xyz789abc123def456ghi", + "note": "Commented backup key", + "severity": "high" + }, + { + "id": 23, + "file": "src/obfuscated.py", + "line": 7, + "difficulty": "hard", + "type": "stripe_key_rot13", + "value": "fx_yvir_frperg_xrl_12345", + "decoded": "sk_live_secret_key_12345", + "severity": "critical" + }, + { + "id": 24, + "file": "src/obfuscated.py", + "line": 10, + "difficulty": "hard", + "type": "github_token_binary", + "value": "b'\\x67\\x68\\x70\\x5f\\x4d\\x79\\x47\\x69\\x74\\x48\\x75\\x62\\x54\\x6f\\x6b\\x65\\x6e\\x31\\x32\\x33\\x34\\x35\\x36'", + "decoded": "ghp_MyGitHubToken123456", + "severity": "critical" + }, + { + "id": 25, + "file": "src/obfuscated.py", + "line": 13, + "difficulty": "hard", + "type": "aws_secret_char_array", + "value": "['A','W','S','_','S','E','C','R','E','T','_','K','E','Y','_','X','Y','Z','7','8','9']", + "decoded": "AWS_SECRET_KEY_XYZ789", + "severity": "critical" + }, + { + "id": 26, + "file": "src/obfuscated.py", + "line": 17, + "difficulty": "hard", + "type": "api_token_reversed", + "value": "321cba_desrever_nekot_ipa", + "decoded": "api_token_reversed_abc123", + "severity": "high" + }, + { + "id": 27, + "file": "src/advanced.js", + "line": 4, + "difficulty": "hard", + "type": "secret_template_string", + "value": "sk_prod_template_key_xyz", + "note": "Built from template literals", + "severity": "critical" + }, + { + "id": 28, + "file": "src/advanced.js", + "line": 7, + "difficulty": "hard", + "type": "password_in_regex", + "value": "password_regex_secret_789", + "note": "Inside regex pattern", + "severity": "medium" + }, + { + "id": 29, + "file": "src/advanced.js", + "line": 10, + "difficulty": "hard", + "type": "api_key_xor", + "value": "[65,82,90,75,94,91,92,75,93,67,65,90,67,92,75,91,67,95]", + "decoded": "api_xor_secret_key", + "note": "XOR encrypted with key 42", + "severity": "critical" + }, + { + "id": 30, + "file": "src/advanced.js", + "line": 17, + "difficulty": "hard", + "type": "api_key_escaped_json", + "value": "sk_escaped_json_key_456", + "note": "Escaped JSON within string", + "severity": "high" + }, + { + "id": 31, + "file": "src/Crypto.go", + "line": 10, + "difficulty": "hard", + "type": "secret_in_heredoc", + "value": "golang_heredoc_secret_999", + "note": "In heredoc/multi-line string", + "severity": "high" + }, + { + "id": 32, + "file": "src/Crypto.go", + "line": 15, + "difficulty": "hard", + "type": "stripe_key_typo", + "value": "strippe_sk_live_corrected_key", + "decoded": "stripe_sk_live_corrected_key", + "note": "Intentional typo corrected programmatically", + "severity": "critical" + } + ], + "file_summary": { + ".env": 2, + "config/settings.py": 3, + "src/app.py": 1, + "config/database.yaml": 1, + "scripts/webhook.js": 1, + "config/app.properties": 1, + "id_rsa": 1, + "config/oauth.json": 1, + "src/Main.java": 1, + "src/config.py": 3, + "scripts/deploy.sh": 2, + "config/keys.yaml": 2, + "src/database.sql": 1, + "config/legacy.ini": 2, + "src/obfuscated.py": 4, + "src/advanced.js": 4, + "src/Crypto.go": 2 + }, + "notes": { + "easy_secrets": "Standard patterns that any decent secret scanner should detect", + "medium_secrets": "Slightly obfuscated - base64, hex, concatenated, or in comments", + "hard_secrets": "Well hidden - ROT13, binary, XOR, reversed, split across constructs" + } +} diff --git a/test_projects/secret_detection_benchmark/.env b/test_projects/secret_detection_benchmark/.env new file mode 100644 index 0000000..ac08a7c --- /dev/null +++ b/test_projects/secret_detection_benchmark/.env @@ -0,0 +1,7 @@ +# Environment configuration +# EASY SECRET #1: Plain AWS access key +AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +DATABASE_HOST=localhost +DATABASE_PORT=5432 diff --git a/test_projects/secret_detection_benchmark/.fuzzforge/findings.db b/test_projects/secret_detection_benchmark/.fuzzforge/findings.db new file mode 100644 index 0000000000000000000000000000000000000000..8bc12cad7985891ed3cc48d93f91ba77430871b7 GIT binary patch literal 53248 zcmeI*Pfy!s90zbaPD2x*Znb5R)={iZRmlnj0||e6X%^fHH3SxdM7K%SL;M6h5(n%j zqb=f4)Lo{%gzZqN+N8C6PEyF!>UC0rH)0pX6K2!bH-&mjN!Q<%>L{RRFlw$1mNm4xxH-yM|y z5eC9{giGJZKM#^Yx&OQV<;cI0XOT?!&+xZAfC&N+fB*y_009V`A%RZ^La`e+#7B&5 zl&P~@=6{*e9Y)x6`}|-gucZr`Qb>QC)s*(g2-_%+n#RLXi|&qEWj1OV{D(s-J7pTb z-(S!D+0oo;TLyKYS8lbL+bvIgJratkS4GFP4Em5CQ7_uW*`kha*DT#M8cY3+Y>T-C zZCyFOnX;Ty8k;I3-k$N#!=c#Jkhs^(T;g-4wdtq!7bC}V`;kT&+~oeMX7=@qJbSb) zJ3p$!3r`87hEABf7(SNy$VP5vca<{I+A*H*3&j#qvDz$meW$&uy&pys$4`?_cJHfZJZmg9%j3w^PxSH-WA^)AqWZ+wLssk~0r zY?O%7m`p=i%oVhIT3%VnFD|9?50v}b10}sySYFKWeV4RcVYIn5FC6a$N=_Rv#G~m!Lsco`HUO5)DyXm!T zLAhC>hFPoJY^7#qh4S>39i_0iq^%axODkPUeRnypE#Ax3tL=AIC9mDp@>(vVtu}gu zSMzvXBEAyj?RQIj&~eJq-5Rxu&9l3)c%j4JKP4N%5sOsls6%(-ha*y7Y-mV)`e8k% z-@pCuU`G~ztuoSVn3HTqhuCIGHwYuW8rrE5lC$ zx_`{N1AJ+4N02|dRN!BjAOHafKmY;|fB*y_009U<;DiL85Gj&Tho5A|(&;OkCM#32 z?dS#{@)em`wZ>*QCT2<-b2G<7ndPOGtX9w#5^D}0Y$ljd+u{Sya$=!ctW~JRoW$MQ zib|5F{0*GiompPrdZP0k#v6^|q*X4JV^w^rgSYn7s@(groAPe-jL=H@5o#%I)t z+2oP7YDA`|)Y+;1g9z`}lKf1Nf0LidKb%k}M;Zt~00Izz00bZa0SG_<0uX=z1YS>I zSjzBUC+c~hNa_*y@391~Nttk${r$TY^W)X(0ds2tWV=5P$##AOHafKmY>gTwqWd7Ml-@lsc9I z1JZD~8S}>fF9i87`4{=cIafpVApijgKmY;|fB*y_009U<00O5_;5|w0@@PP*`|3bc zQoB3|&>R0h7v$gNpXBGKu#zYe1Rwwb2tWV=5P$##AOHafK;WDRT$9q`u{RSo-^3_& z4L4tD7#Nb$;m*C=-wo)F{{=k%|D5zBR1*RafB*y_009U<00Izz00hpf0LK64wdGN7 z2tWV=5P$##AOHafKmY;|I8OrZ^Z$eLp&Wmg`N*FbC!gw6-|9cBfij)w500bZa0SG_<0uX=z1Rwx` HQzq~qGNgZB literal 0 HcmV?d00001 diff --git a/test_projects/secret_detection_benchmark/README.md b/test_projects/secret_detection_benchmark/README.md new file mode 100644 index 0000000..2101179 --- /dev/null +++ b/test_projects/secret_detection_benchmark/README.md @@ -0,0 +1,99 @@ +# Secret Detection Benchmark Dataset + +Ground truth dataset with **exactly 32 known secrets** for testing secret detection tools. + +## Contents + +- **12 Easy Secrets**: Standard patterns (AWS keys, GitHub PATs, Stripe keys, etc.) +- **10 Medium Secrets**: Slightly obfuscated (Base64, hex, concatenated, in comments) +- **10 Hard Secrets**: Well hidden (ROT13, binary, XOR, reversed, template strings) + +## Files + +``` +├── .env # 2 secrets +├── config/ +│ ├── settings.py # 3 secrets +│ ├── database.yaml # 1 secret +│ ├── app.properties # 1 secret +│ ├── oauth.json # 1 secret +│ ├── keys.yaml # 2 secrets +│ └── legacy.ini # 2 secrets +├── src/ +│ ├── app.py # 1 secret +│ ├── Main.java # 1 secret +│ ├── config.py # 3 secrets (medium difficulty) +│ ├── obfuscated.py # 4 secrets (hard difficulty) +│ ├── advanced.js # 4 secrets (hard difficulty) +│ ├── Crypto.go # 2 secrets (hard difficulty) +│ └── database.sql # 1 secret +├── scripts/ +│ ├── webhook.js # 1 secret +│ └── deploy.sh # 2 secrets +└── id_rsa # 1 secret + +Total: 17 files with 32 secrets +``` + +## Secret Difficulty Breakdown + +### Easy (12 secrets) +Should be detected by any decent secret scanner: +- Plain AWS access keys +- GitHub Personal Access Tokens +- Stripe API keys +- Database passwords in plain text +- JWT secrets +- SSH private keys +- OAuth secrets +- Slack webhooks + +### Medium (10 secrets) +Requires some parsing or contextual understanding: +- Base64 encoded AWS key +- Hex-encoded tokens +- Split strings concatenated at runtime +- URL-encoded passwords +- Multi-line private keys in YAML +- Secrets with Unicode characters +- Secrets in SQL/shell comments +- Deprecated config formats + +### Hard (10 secrets) +Well hidden, may challenge even advanced tools: +- ROT13 encoded secrets +- Binary string representations +- Character array joins +- Reversed strings +- Template string constructs +- Secrets in regex patterns +- XOR encrypted values +- Escaped JSON within strings +- Heredoc patterns +- Intentional typos corrected programmatically + +## Usage + +Run secret detection tools against this directory and compare results to the ground truth file (located in `backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json`) to calculate: + +- **Precision**: TP / (TP + FP) - How many detected secrets are real? +- **Recall**: TP / (TP + FN) - How many real secrets were found? +- **F1 Score**: 2 × (Precision × Recall) / (Precision + Recall) + +### Expected Performance + +| Tool Type | Expected Easy | Expected Medium | Expected Hard | Total Expected | +|-----------|---------------|-----------------|---------------|----------------| +| Pattern-based (Gitleaks) | 12/12 (100%) | 6-8/10 (60-80%) | 2-4/10 (20-40%) | 20-24/32 | +| Entropy-based (TruffleHog) | 12/12 (100%) | 5-7/10 (50-70%) | 1-3/10 (10-30%) | 18-22/32 | +| LLM-based | 12/12 (100%) | 8-10/10 (80-100%) | 4-8/10 (40-80%) | 24-30/32 | + +## Validation + +Use the validation script to check tool performance: + +```bash +python validate_ground_truth.py --tool-output results.json +``` + +This will calculate precision, recall, and F1 score against the ground truth. diff --git a/test_projects/secret_detection_benchmark/config/app.properties b/test_projects/secret_detection_benchmark/config/app.properties new file mode 100644 index 0000000..d9b2ece --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/app.properties @@ -0,0 +1,9 @@ +# Application properties file +app.name=SecretDetectionBenchmark +app.version=1.0.0 + +# EASY SECRET #8: API Key +api.key=sk_test_4eC39HqLyjWDarjtT1zdp7dc +api.endpoint=https://api.example.com + +logging.level=INFO diff --git a/test_projects/secret_detection_benchmark/config/database.yaml b/test_projects/secret_detection_benchmark/config/database.yaml new file mode 100644 index 0000000..d211c6d --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/database.yaml @@ -0,0 +1,10 @@ +# Database configuration +databases: + production: + host: prod-db.example.com + port: 5432 + # EASY SECRET #6: Azure connection string + connection_string: "DefaultEndpointsProtocol=https;AccountName=prodstore;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;EndpointSuffix=core.windows.net" + staging: + host: staging-db.example.com + port: 5432 diff --git a/test_projects/secret_detection_benchmark/config/keys.yaml b/test_projects/secret_detection_benchmark/config/keys.yaml new file mode 100644 index 0000000..90d1009 --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/keys.yaml @@ -0,0 +1,12 @@ +# Keys configuration +api_keys: + production: + # MEDIUM SECRET #16: Multi-line private key in YAML literal block + private_key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAyLqJZvd5CZxJhLZYLFCqLV9G5k8dFz1LoNwPPfK3qE1k8H4y + FQwNyX3WJZNmKJLOPQMfHZQxGhHJPwZYjKQPYHJ1234567890abcdefghijklmno + -----END RSA PRIVATE KEY----- + + # MEDIUM SECRET #17: Secret with Unicode characters + api_token_intl: "tøkęn_śęçrėt_ẃïth_ŭñïçődė_123456" diff --git a/test_projects/secret_detection_benchmark/config/legacy.ini b/test_projects/secret_detection_benchmark/config/legacy.ini new file mode 100644 index 0000000..7a479ec --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/legacy.ini @@ -0,0 +1,8 @@ +[database] +; MEDIUM SECRET #19: Secret in deprecated INI format +password = L3g@cy_DB_P@ssw0rd_2023 + +[api] +; MEDIUM SECRET #20: Commented backup API key +; old_api_key = backup_key_xyz789abc123def456ghi +endpoint = https://api.legacy.example.com diff --git a/test_projects/secret_detection_benchmark/config/oauth.json b/test_projects/secret_detection_benchmark/config/oauth.json new file mode 100644 index 0000000..7fb676b --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/oauth.json @@ -0,0 +1,11 @@ +{ + "oauth_provider": "google", + "client_id": "123456789012-abcdefghijklmnopqrstuvwxyz123456.apps.googleusercontent.com", + "client_secret": "GOCSPX-Ab12Cd34Ef56Gh78Ij90Kl12", + "redirect_uri": "https://example.com/oauth/callback", + "scopes": [ + "openid", + "email", + "profile" + ] +} diff --git a/test_projects/secret_detection_benchmark/config/settings.py b/test_projects/secret_detection_benchmark/config/settings.py new file mode 100644 index 0000000..1f5986a --- /dev/null +++ b/test_projects/secret_detection_benchmark/config/settings.py @@ -0,0 +1,21 @@ +""" +Application settings and configuration +""" + +# EASY SECRET #2: GitHub Personal Access Token +GITHUB_TOKEN = "ghp_vR8jK2mN4pQ6tX9bC3wY7zA1eF5hI8kL" + +# EASY SECRET #3: Stripe API key +STRIPE_SECRET_KEY = "sk_live_51MabcdefghijklmnopqrstuvwxyzABCDEF123456789" + +# Application settings +DEBUG = False +LOG_LEVEL = "INFO" + +# EASY SECRET #4: Database password +DATABASE_CONFIG = { + "host": "prod-db.example.com", + "port": 5432, + "username": "admin", + "password": "ProdDB_P@ssw0rd_2024_Secure!" +} diff --git a/test_projects/secret_detection_benchmark/id_rsa b/test_projects/secret_detection_benchmark/id_rsa new file mode 100644 index 0000000..4f7c8b0 --- /dev/null +++ b/test_projects/secret_detection_benchmark/id_rsa @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAyLqJZvd5CZxJhLZYLFCqLV9G5k8dFz1LoNwPPfK3qE1k8H4yFQwN +yX3WJZNmKJLOPQMfHZQxGhHJPwZYjKQPYHJ1oNwPPfK3qE1k8H4yFQwNyX3WJZNmKJLO +PQMfHZQxGhHJPwZYjKQPYHJ1aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa== +-----END OPENSSH PRIVATE KEY----- diff --git a/test_projects/secret_detection_benchmark/scripts/deploy.sh b/test_projects/secret_detection_benchmark/scripts/deploy.sh new file mode 100644 index 0000000..48c8eca --- /dev/null +++ b/test_projects/secret_detection_benchmark/scripts/deploy.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Deployment script + +# MEDIUM SECRET #14: Secret in environment variable export +export SECRET_API_KEY="sk_prod_1234567890abcdefghijklmnopqrstuvwxyz" + +echo "Deploying application..." + +# MEDIUM SECRET #15: URL-encoded secret in connection string (backup comment) +# backup_connection="mysql://admin:MyP%40ssw0rd%21@db.example.com:3306/prod" + +deploy_app() { + echo "Deployment complete" +} + +deploy_app diff --git a/test_projects/secret_detection_benchmark/scripts/webhook.js b/test_projects/secret_detection_benchmark/scripts/webhook.js new file mode 100644 index 0000000..aeba3f0 --- /dev/null +++ b/test_projects/secret_detection_benchmark/scripts/webhook.js @@ -0,0 +1,13 @@ +// Webhook configuration and handlers + +// EASY SECRET #7: Slack webhook URL +const SLACK_WEBHOOK = "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXX"; + +function sendSlackNotification(message) { + fetch(SLACK_WEBHOOK, { + method: 'POST', + body: JSON.stringify({ text: message }) + }); +} + +module.exports = { sendSlackNotification }; diff --git a/test_projects/secret_detection_benchmark/src/Crypto.go b/test_projects/secret_detection_benchmark/src/Crypto.go new file mode 100644 index 0000000..14d2266 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/Crypto.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "strings" +) + +// HARD SECRET #29: Heredoc with unusual delimiter +const ConfigTemplate = ` +SECRET_KEY=golang_heredoc_secret_999 +END_OF_CONFIG +` + +// HARD SECRET #30: Secret with intentional typo corrected programmatically +const API_KEY_TYPO = "strippe_sk_live_corrected_key" + +func CorrectTypo(s string) string { + return strings.Replace(s, "strippe", "stripe", 1) +} + +func main() { + fmt.Println("Crypto utilities initialized") + correctedKey := CorrectTypo(API_KEY_TYPO) + fmt.Println("Key ready:", correctedKey[:10]+"...") +} diff --git a/test_projects/secret_detection_benchmark/src/Main.java b/test_projects/secret_detection_benchmark/src/Main.java new file mode 100644 index 0000000..9a90be7 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/Main.java @@ -0,0 +1,10 @@ +package com.example.benchmark; + +public class Main { + // EASY SECRET #10: Google OAuth secret in Java + private static final String GOOGLE_OAUTH_SECRET = "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"; + + public static void main(String[] args) { + System.out.println("Application starting..."); + } +} diff --git a/test_projects/secret_detection_benchmark/src/advanced.js b/test_projects/secret_detection_benchmark/src/advanced.js new file mode 100644 index 0000000..6404ce2 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/advanced.js @@ -0,0 +1,19 @@ +// Advanced obfuscation techniques + +// HARD SECRET #25: Template string with escaping +const SECRET_TEMPLATE = `sk_${"prod"}_${"template"}_${"key"}_xyz`; + +// HARD SECRET #26: Secret in regex pattern +const PASSWORD_REGEX = /password_regex_secret_789/; + +// HARD SECRET #27: XORed secret (XOR with key 42) +const XOR_SECRET = [65,82,90,75,94,91,92,75,93,67,65,90,67,92,75,91,67,95]; + +function decodeXOR() { + return String.fromCharCode(...XOR_SECRET.map(c => c ^ 42)); +} + +// HARD SECRET #28: Escaped JSON within string +const CONFIG_JSON = "{\"api_key\":\"sk_escaped_json_key_456\"}"; + +module.exports = { SECRET_TEMPLATE, decodeXOR }; diff --git a/test_projects/secret_detection_benchmark/src/app.py b/test_projects/secret_detection_benchmark/src/app.py new file mode 100644 index 0000000..5d17ade --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/app.py @@ -0,0 +1,19 @@ +""" +Main application entry point +""" +import os + +# EASY SECRET #5: JWT Secret +JWT_SECRET_KEY = "my-super-secret-jwt-key-do-not-share-2024" + +def init_app(): + """Initialize the application""" + app_config = { + "name": "SecretDetectionBenchmark", + "version": "1.0.0" + } + return app_config + +if __name__ == "__main__": + print("Application starting...") + init_app() diff --git a/test_projects/secret_detection_benchmark/src/config.py b/test_projects/secret_detection_benchmark/src/config.py new file mode 100644 index 0000000..22ab19a --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/config.py @@ -0,0 +1,19 @@ +""" +Configuration with moderately obfuscated secrets +""" +import base64 + +# MEDIUM SECRET #11: Base64 encoded AWS key +AWS_KEY_ENCODED = "QUtJQUlPU0ZPRE5ON0VYQU1QTEU=" + +# MEDIUM SECRET #12: Hex-encoded API token +HEX_TOKEN = "6170695f746f6b656e5f616263313233787977373839" + +# MEDIUM SECRET #13: Split secret concatenated at runtime +DB_PASS_PART1 = "MySecure" +DB_PASS_PART2 = "Password" +DB_PASS_PART3 = "2024!" +DATABASE_PASSWORD = DB_PASS_PART1 + DB_PASS_PART2 + DB_PASS_PART3 + +def get_aws_key(): + return base64.b64decode(AWS_KEY_ENCODED).decode() diff --git a/test_projects/secret_detection_benchmark/src/database.sql b/test_projects/secret_detection_benchmark/src/database.sql new file mode 100644 index 0000000..e43eac7 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/database.sql @@ -0,0 +1,15 @@ +-- Database initialization script + +CREATE DATABASE prod_db; + +-- MEDIUM SECRET #18: Secret in SQL comment +-- Connection string: postgresql://admin:Pr0dDB_S3cr3t_P@ss@db.prod.example.com:5432/prod_db + +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + username VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL +); + +-- Insert test data +INSERT INTO users (username, email) VALUES ('admin', 'admin@example.com'); diff --git a/test_projects/secret_detection_benchmark/src/obfuscated.py b/test_projects/secret_detection_benchmark/src/obfuscated.py new file mode 100644 index 0000000..ead77d4 --- /dev/null +++ b/test_projects/secret_detection_benchmark/src/obfuscated.py @@ -0,0 +1,23 @@ +""" +Heavily obfuscated secrets - hard to detect +""" +import codecs + +# HARD SECRET #21: ROT13 encoded secret +SECRET_ROT13 = "fx_yvir_frperg_xrl_12345" + +# HARD SECRET #22: Binary string representation +GITHUB_TOKEN_BYTES = b'\x67\x68\x70\x5f\x4d\x79\x47\x69\x74\x48\x75\x62\x54\x6f\x6b\x65\x6e\x31\x32\x33\x34\x35\x36' + +# HARD SECRET #23: Character array join +AWS_SECRET_CHARS = ['A','W','S','_','S','E','C','R','E','T','_','K','E','Y','_','X','Y','Z','7','8','9'] +AWS_SECRET = ''.join(AWS_SECRET_CHARS) + +# HARD SECRET #24: Reversed string that's un-reversed at runtime +TOKEN_REVERSED = "321cba_desrever_nekot_ipa" + +def get_rot13_secret(): + return codecs.decode(SECRET_ROT13, 'rot_13') + +def get_token(): + return TOKEN_REVERSED[::-1] diff --git a/test_projects/secret_detection_benchmark/validate_ground_truth.py b/test_projects/secret_detection_benchmark/validate_ground_truth.py new file mode 100644 index 0000000..958e21c --- /dev/null +++ b/test_projects/secret_detection_benchmark/validate_ground_truth.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +""" +Validate secret detection tool results against ground truth +""" +import json +import argparse +from pathlib import Path +from typing import Set, Tuple + +def load_ground_truth(ground_truth_file: Path) -> Set[Tuple[str, int]]: + """Load ground truth secrets as set of (file, line) tuples""" + with open(ground_truth_file) as f: + data = json.load(f) + + secrets = set() + for secret in data["secrets"]: + secrets.add((secret["file"], secret["line"])) + + return secrets + +def load_tool_results(results_file: Path) -> Set[Tuple[str, int]]: + """Load tool results as set of (file, line) tuples""" + with open(results_file) as f: + data = json.load(f) + + findings = set() + # Assume SARIF format or custom format with findings_by_file + if "findings_by_file" in data: + for file_path, lines in data["findings_by_file"].items(): + for line in lines: + findings.add((file_path, line)) + + return findings + +def calculate_metrics(ground_truth: Set, detected: Set): + """Calculate precision, recall, and F1 score""" + tp = len(ground_truth & detected) # True positives + fp = len(detected - ground_truth) # False positives + fn = len(ground_truth - detected) # False negatives + + precision = tp / (tp + fp) if (tp + fp) > 0 else 0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 0 + f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0 + + return { + "true_positives": tp, + "false_positives": fp, + "false_negatives": fn, + "precision": precision * 100, + "recall": recall * 100, + "f1_score": f1 * 100 + } + +def main(): + parser = argparse.ArgumentParser(description="Validate tool results against ground truth") + parser.add_argument("--tool-output", required=True, help="Path to tool output JSON") + parser.add_argument("--ground-truth", + default="../../backend/benchmarks/by_category/secret_detection/secret_detection_benchmark_GROUND_TRUTH.json", + help="Path to ground truth file") + args = parser.parse_args() + + ground_truth = load_ground_truth(Path(args.ground_truth)) + detected = load_tool_results(Path(args.tool_output)) + metrics = calculate_metrics(ground_truth, detected) + + print("\n" + "="*60) + print("Secret Detection Validation Results") + print("="*60) + print(f"Ground Truth Secrets: {len(ground_truth)}") + print(f"Detected Secrets: {len(detected)}") + print(f"\nTrue Positives: {metrics['true_positives']}") + print(f"False Positives: {metrics['false_positives']}") + print(f"False Negatives: {metrics['false_negatives']}") + print(f"\n{'Precision:':<15} {metrics['precision']:.2f}%") + print(f"{'Recall:':<15} {metrics['recall']:.2f}%") + print(f"{'F1 Score:':<15} {metrics['f1_score']:.2f}%") + print("="*60 + "\n") + +if __name__ == "__main__": + main()