diff --git a/.gitignore b/.gitignore index b090789..a2f9683 100644 --- a/.gitignore +++ b/.gitignore @@ -238,6 +238,7 @@ yarn-error.log* !test_projects/secret_detection_benchmark/ !test_projects/secret_detection_benchmark/** !**/secret_detection_benchmark_GROUND_TRUTH.json +!**/secret_detection/results/ secret* secrets/ diff --git a/backend/benchmarks/by_category/secret_detection/results/comparison_report.md b/backend/benchmarks/by_category/secret_detection/results/comparison_report.md new file mode 100644 index 0000000..220cb33 --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/results/comparison_report.md @@ -0,0 +1,169 @@ +# Secret Detection Tools Comparison + +**Target**: secret_detection_benchmark +**Tools**: Gitleaks, TruffleHog, LLM (gpt-4o-mini), LLM (gpt-5-mini) + + +## Summary + +| Tool | Secrets | Files | Avg/File | Time (s) | +|------|---------|-------|----------|----------| +| Gitleaks | 12 | 10 | 1.2 | 5.18 | +| TruffleHog | 1 | 1 | 1.0 | 5.06 | +| LLM (gpt-4o-mini) | 30 | 15 | 2.0 | 296.85 | +| LLM (gpt-5-mini) | 41 | 16 | 2.6 | 618.55 | + +## Agreement Analysis + +Secrets found by different numbers of tools: + +- **3 tools agree**: 6 secrets +- **2 tools agree**: 22 secrets +- **Only 1 tool found**: 22 secrets + +## Tool Agreement Matrix + +Number of common secrets found by tool pairs: + +| Tool | Gitleaks | TruffleHog | gpt-4o-mini | gpt-5-mini | +|------|------|------|------|------| +| Gitleaks | 12 | 0 | 7 | 11 | +| TruffleHog | 0 | 1 | 0 | 0 | +| gpt-4o-mini | 7 | 0 | 30 | 22 | +| gpt-5-mini | 11 | 0 | 22 | 41 | + +## Per-File Detailed Comparison + +Secrets found per file by each tool: + +| File | Gitleaks | TruffleHog | gpt-4o-mini | gpt-5-mini | Total | +|------|------|------|------|------|------| +| `src/obfuscated.py` | 2 | 0 | 6 | 7 | **15** | +| `src/advanced.js` | 0 | 0 | 5 | 7 | **12** | +| `src/config.py` | 1 | 0 | 0 | 6 | **7** | +| `.env` | 1 | 0 | 2 | 2 | **5** | +| `config/keys.yaml` | 1 | 0 | 2 | 2 | **5** | +| `config/oauth.json` | 1 | 0 | 2 | 2 | **5** | +| `config/settings.py` | 2 | 0 | 0 | 3 | **5** | +| `scripts/deploy.sh` | 1 | 0 | 2 | 2 | **5** | +| `config/legacy.ini` | 0 | 0 | 2 | 2 | **4** | +| `src/Crypto.go` | 0 | 0 | 2 | 2 | **4** | +| `config/app.properties` | 1 | 0 | 1 | 1 | **3** | +| `config/database.yaml` | 0 | 1 | 1 | 1 | **3** | +| `src/Main.java` | 1 | 0 | 1 | 1 | **3** | +| `id_rsa` | 1 | 0 | 1 | 0 | **2** | +| `scripts/webhook.js` | 0 | 0 | 1 | 1 | **2** | +| ... and 2 more files | ... | ... | ... | ... | ... | + +## File Type Breakdown + +| Type | Gitleaks | TruffleHog | gpt-4o-mini | gpt-5-mini | +|------|------|------|------|------| +| `.env` | 1 files | 0 files | 1 files | 1 files | +| `.go` | 0 files | 0 files | 1 files | 1 files | +| `.ini` | 0 files | 0 files | 1 files | 1 files | +| `.java` | 1 files | 0 files | 1 files | 1 files | +| `.js` | 0 files | 0 files | 2 files | 2 files | +| `.json` | 1 files | 0 files | 1 files | 1 files | +| `.properties` | 1 files | 0 files | 1 files | 1 files | +| `.py` | 3 files | 0 files | 2 files | 4 files | +| `.sh` | 1 files | 0 files | 1 files | 1 files | +| `.sql` | 0 files | 0 files | 1 files | 1 files | +| `.yaml` | 1 files | 1 files | 2 files | 2 files | +| `[no extension]` | 1 files | 0 files | 1 files | 0 files | + +## Files Analyzed + +**Total unique files with secrets**: 17 + + +### Gitleaks + +Found secrets in **10 files**: + +- `config/settings.py`: 2 secrets (lines: 6, 9) +- `src/obfuscated.py`: 2 secrets (lines: 7, 17) +- `.env`: 1 secrets (lines: 3) +- `config/app.properties`: 1 secrets (lines: 6) +- `config/keys.yaml`: 1 secrets (lines: 6) +- `id_rsa`: 1 secrets (lines: 1) +- `config/oauth.json`: 1 secrets (lines: 4) +- `scripts/deploy.sh`: 1 secrets (lines: 5) +- `src/Main.java`: 1 secrets (lines: 5) +- `src/config.py`: 1 secrets (lines: 7) + +### TruffleHog + +Found secrets in **1 files**: + +- `config/database.yaml`: 1 secrets (lines: 6) + +### LLM (gpt-4o-mini) + +Found secrets in **15 files**: + +- `src/obfuscated.py`: 6 secrets (lines: 7, 10, 13, 18, 20...) +- `src/advanced.js`: 5 secrets (lines: 4, 7, 10, 12, 17) +- `src/Crypto.go`: 2 secrets (lines: 6, 10) +- `.env`: 2 secrets (lines: 3, 4) +- `config/keys.yaml`: 2 secrets (lines: 6, 12) +- `config/oauth.json`: 2 secrets (lines: 3, 4) +- `config/legacy.ini`: 2 secrets (lines: 4, 7) +- `scripts/deploy.sh`: 2 secrets (lines: 6, 9) +- `src/app.py`: 1 secrets (lines: 7) +- `scripts/webhook.js`: 1 secrets (lines: 4) +- ... and 5 more files + +### LLM (gpt-5-mini) + +Found secrets in **16 files**: + +- `src/obfuscated.py`: 7 secrets (lines: 7, 10, 13, 14, 17...) +- `src/advanced.js`: 7 secrets (lines: 4, 7, 9, 10, 13...) +- `src/config.py`: 6 secrets (lines: 7, 10, 13, 14, 15...) +- `config/settings.py`: 3 secrets (lines: 6, 9, 20) +- `src/Crypto.go`: 2 secrets (lines: 10, 15) +- `.env`: 2 secrets (lines: 3, 4) +- `config/keys.yaml`: 2 secrets (lines: 6, 12) +- `config/oauth.json`: 2 secrets (lines: 3, 4) +- `config/legacy.ini`: 2 secrets (lines: 3, 7) +- `scripts/deploy.sh`: 2 secrets (lines: 5, 10) +- ... and 6 more files + +## Overlap Analysis + + +**No files were found by all tools** + + +## Ground Truth Analysis + +**Expected secrets**: 32 (documented in ground truth) + +### Tool Performance vs Ground Truth + +| Tool | Found | Expected | Recall | Extra Findings | +|------|-------|----------|--------|----------------| +| Gitleaks | 12 | 32 | 37.5% | 0 | +| TruffleHog | 1 | 32 | 0.0% | 1 | +| LLM (gpt-4o-mini) | 30 | 32 | 56.2% | 12 | +| LLM (gpt-5-mini) | 41 | 32 | 84.4% | 14 | + +### LLM Extra Findings Explanation + +LLMs may find more than 30 secrets because they detect: + +- **Split secret components**: Each part of `DB_PASS_PART1 + PART2 + PART3` counted separately +- **Join operations**: Lines like `''.join(AWS_SECRET_CHARS)` flagged as additional exposure +- **Decoding functions**: Code that reveals secrets (e.g., `base64.b64decode()`, `codecs.decode()`) +- **Comment identifiers**: Lines marking secret locations without plaintext values + +These are *technically correct* detections of secret exposure points, not false positives. +The ground truth documents 30 'primary' secrets, but the codebase has additional derivative exposures. + + +## Performance Summary + +- **Most secrets found**: LLM (gpt-5-mini) (41 secrets) +- **Most files covered**: LLM (gpt-5-mini) (16 files) +- **Fastest**: TruffleHog (5.06s) \ No newline at end of file diff --git a/backend/benchmarks/by_category/secret_detection/results/comparison_results.json b/backend/benchmarks/by_category/secret_detection/results/comparison_results.json new file mode 100644 index 0000000..4e9c89f --- /dev/null +++ b/backend/benchmarks/by_category/secret_detection/results/comparison_results.json @@ -0,0 +1,253 @@ +{ + "target_path": "/Users/tduhamel/Documents/FuzzingLabs/fuzzforge_ai/test_projects/secret_detection_benchmark", + "results": [ + { + "tool_name": "Gitleaks", + "execution_time": 5.177123069763184, + "findings_count": 12, + "findings_by_file": { + ".env": [ + 3 + ], + "config/app.properties": [ + 6 + ], + "config/keys.yaml": [ + 6 + ], + "id_rsa": [ + 1 + ], + "config/oauth.json": [ + 4 + ], + "scripts/deploy.sh": [ + 5 + ], + "config/settings.py": [ + 6, + 9 + ], + "src/Main.java": [ + 5 + ], + "src/obfuscated.py": [ + 7, + 17 + ], + "src/config.py": [ + 7 + ] + }, + "unique_files": 10, + "unique_locations": 12, + "secret_density": 1.2, + "file_types": { + ".env": 1, + ".properties": 1, + ".yaml": 1, + "[no extension]": 1, + ".json": 1, + ".sh": 1, + ".py": 3, + ".java": 1 + } + }, + { + "tool_name": "TruffleHog", + "execution_time": 5.061383008956909, + "findings_count": 1, + "findings_by_file": { + "config/database.yaml": [ + 6 + ] + }, + "unique_files": 1, + "unique_locations": 1, + "secret_density": 1.0, + "file_types": { + ".yaml": 1 + } + }, + { + "tool_name": "LLM (gpt-4o-mini)", + "execution_time": 296.8492441177368, + "findings_count": 30, + "findings_by_file": { + "src/obfuscated.py": [ + 7, + 10, + 13, + 18, + 20, + 23 + ], + "src/app.py": [ + 7 + ], + "scripts/webhook.js": [ + 4 + ], + "src/advanced.js": [ + 4, + 7, + 10, + 12, + 17 + ], + "src/Main.java": [ + 5 + ], + "src/Crypto.go": [ + 6, + 10 + ], + ".env": [ + 3, + 4 + ], + "config/keys.yaml": [ + 6, + 12 + ], + "config/database.yaml": [ + 7 + ], + "config/oauth.json": [ + 3, + 4 + ], + "config/legacy.ini": [ + 4, + 7 + ], + "src/database.sql": [ + 4 + ], + "config/app.properties": [ + 6 + ], + "scripts/deploy.sh": [ + 6, + 9 + ], + "id_rsa": [ + 1 + ] + }, + "unique_files": 15, + "unique_locations": 30, + "secret_density": 2.0, + "file_types": { + ".py": 2, + ".js": 2, + ".java": 1, + ".go": 1, + ".env": 1, + ".yaml": 2, + ".json": 1, + ".ini": 1, + ".sql": 1, + ".properties": 1, + ".sh": 1, + "[no extension]": 1 + } + }, + { + "tool_name": "LLM (gpt-5-mini)", + "execution_time": 618.5462851524353, + "findings_count": 41, + "findings_by_file": { + "config/settings.py": [ + 6, + 9, + 20 + ], + "src/obfuscated.py": [ + 7, + 10, + 13, + 14, + 17, + 20, + 23 + ], + "src/app.py": [ + 7 + ], + "src/config.py": [ + 7, + 10, + 13, + 14, + 15, + 16 + ], + "scripts/webhook.js": [ + 4 + ], + "src/advanced.js": [ + 4, + 7, + 9, + 10, + 13, + 17, + 19 + ], + "src/Main.java": [ + 5 + ], + "src/Crypto.go": [ + 10, + 15 + ], + ".env": [ + 3, + 4 + ], + "config/keys.yaml": [ + 6, + 12 + ], + "config/database.yaml": [ + 7 + ], + "config/oauth.json": [ + 3, + 4 + ], + "config/legacy.ini": [ + 3, + 7 + ], + "src/database.sql": [ + 6 + ], + "config/app.properties": [ + 6 + ], + "scripts/deploy.sh": [ + 5, + 10 + ] + }, + "unique_files": 16, + "unique_locations": 41, + "secret_density": 2.5625, + "file_types": { + ".py": 4, + ".js": 2, + ".java": 1, + ".go": 1, + ".env": 1, + ".yaml": 2, + ".json": 1, + ".ini": 1, + ".sql": 1, + ".properties": 1, + ".sh": 1 + } + } + ] +} \ No newline at end of file