Initial commit: TMT lightweight threat modeling toolkit

tests/test_runner.py

@@ -0,0 +1,225 @@
+"""Test suite for the threat model runner and report generation.
+
+Validates the end-to-end workflow: scanner orchestration, report
+assembly, statistics computation, and file output generation.
+"""
+
+import json
+import os
+import tempfile
+import pytest
+
+from tmt.config import TMTConfig, ScannerConfig, LLMConfig, ReportConfig
+from tmt.models import (
+    Finding,
+    FindingCategory,
+    ScanResult,
+    Severity,
+    ThreatModelReport,
+    compute_report_statistics,
+)
+from tmt.reports.generator import ReportGenerator
+from tmt.runner import ThreatModelRunner
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Path constants for test fixtures
+# ──────────────────────────────────────────────────────────────────────────────
+
+FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
+
+
+def _make_test_config(output_dir: str) -> TMTConfig:
+    """Create a TMTConfig tailored for testing with output to a temp dir.
+
+    Args:
+        output_dir: Temporary directory for report output.
+
+    Returns:
+        TMTConfig with scanning enabled and LLM disabled.
+    """
+    return TMTConfig(
+        project_name="test-project",
+        target_dirs=[FIXTURES_DIR],
+        file_extensions=[".py"],
+        exclude_dirs=["__pycache__", ".git"],
+        scanner=ScannerConfig(enabled=True),
+        llm=LLMConfig(enabled=False),
+        report=ReportConfig(output_dir=output_dir, formats=["markdown", "json"]),
+    )
+
+
+def _make_sample_finding(severity: Severity = Severity.HIGH) -> Finding:
+    """Create a sample Finding for report generation tests.
+
+    Args:
+        severity: Severity level for the sample finding.
+
+    Returns:
+        Finding with test data populated.
+    """
+    return Finding(
+        title="Test Finding",
+        description="A test vulnerability description",
+        severity=severity,
+        category=FindingCategory.AUTH_SESSION,
+        file_path="test.py",
+        line_number=10,
+        code_snippet="vulnerable_code()",
+        recommendation="Fix the vulnerability",
+        confidence=0.9,
+        cwe_id="CWE-000",
+    )
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Report statistics tests
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+class TestReportStatistics:
+    """Test suite for report statistics computation."""
+
+    def test_compute_empty_report(self):
+        """Verify empty report has zero counts."""
+        report = ThreatModelReport(project_name="test")
+        report = compute_report_statistics(report)
+        assert report.total_findings == 0
+        assert report.critical_count == 0
+
+    def test_compute_with_findings(self):
+        """Verify statistics correctly count findings by severity."""
+        scan_result = ScanResult(
+            scanner_name="TestScanner",
+            findings=[
+                _make_sample_finding(Severity.CRITICAL),
+                _make_sample_finding(Severity.CRITICAL),
+                _make_sample_finding(Severity.HIGH),
+                _make_sample_finding(Severity.MEDIUM),
+                _make_sample_finding(Severity.LOW),
+            ],
+        )
+        report = ThreatModelReport(project_name="test", scan_results=[scan_result])
+        report = compute_report_statistics(report)
+        assert report.total_findings == 5
+        assert report.critical_count == 2
+        assert report.high_count == 1
+        assert report.medium_count == 1
+        assert report.low_count == 1
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Report generation tests
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+class TestReportGenerator:
+    """Test suite for Markdown and JSON report file generation."""
+
+    def test_generates_markdown_file(self):
+        """Verify Markdown report file is created with correct content."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = ReportConfig(output_dir=tmpdir, formats=["markdown"])
+            generator = ReportGenerator(config)
+            report = ThreatModelReport(project_name="md-test")
+            paths = generator.generate(report)
+            assert len(paths) == 1
+            assert paths[0].endswith(".md")
+            assert os.path.exists(paths[0])
+
+    def test_generates_json_file(self):
+        """Verify JSON report file is created with valid JSON content."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = ReportConfig(output_dir=tmpdir, formats=["json"])
+            generator = ReportGenerator(config)
+            report = ThreatModelReport(project_name="json-test")
+            paths = generator.generate(report)
+            assert len(paths) == 1
+            with open(paths[0]) as f:
+                data = json.load(f)
+            assert data["project_name"] == "json-test"
+
+    def test_generates_both_formats(self):
+        """Verify both Markdown and JSON files are generated together."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = ReportConfig(output_dir=tmpdir, formats=["markdown", "json"])
+            generator = ReportGenerator(config)
+            report = ThreatModelReport(project_name="dual-test")
+            paths = generator.generate(report)
+            assert len(paths) == 2
+
+    def test_markdown_includes_findings(self):
+        """Verify Markdown report includes finding details."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = ReportConfig(output_dir=tmpdir, formats=["markdown"])
+            generator = ReportGenerator(config)
+            finding = _make_sample_finding()
+            scan_result = ScanResult(scanner_name="TestScanner", findings=[finding])
+            report = ThreatModelReport(
+                project_name="detail-test", scan_results=[scan_result]
+            )
+            paths = generator.generate(report)
+            content = open(paths[0]).read()
+            assert "Test Finding" in content
+            assert "CWE-000" in content
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# End-to-end runner tests
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+class TestThreatModelRunner:
+    """Test suite for the end-to-end threat modeling workflow."""
+
+    def test_runner_produces_report(self):
+        """Verify runner completes and returns a populated report."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = _make_test_config(tmpdir)
+            runner = ThreatModelRunner(config)
+            report = runner.run(target_path=FIXTURES_DIR)
+            assert isinstance(report, ThreatModelReport)
+            assert len(report.scan_results) == 5
+
+    def test_runner_generates_report_files(self):
+        """Verify runner writes report files to the output directory."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = _make_test_config(tmpdir)
+            runner = ThreatModelRunner(config)
+            runner.run(target_path=FIXTURES_DIR)
+            md_path = os.path.join(tmpdir, "threat_model_report.md")
+            json_path = os.path.join(tmpdir, "threat_model_report.json")
+            assert os.path.exists(md_path), "Markdown report should exist"
+            assert os.path.exists(json_path), "JSON report should exist"
+
+    def test_runner_detects_vulnerabilities(self):
+        """Verify runner finds vulnerabilities in the vulnerable fixture."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = _make_test_config(tmpdir)
+            runner = ThreatModelRunner(config)
+            report = runner.run(target_path=FIXTURES_DIR)
+            report = compute_report_statistics(report)
+            assert (
+                report.total_findings > 0
+            ), "Should find vulnerabilities in test fixtures"
+
+    def test_runner_with_llm_disabled(self):
+        """Verify runner works correctly when LLM review is disabled."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = _make_test_config(tmpdir)
+            config.llm.enabled = False
+            runner = ThreatModelRunner(config)
+            report = runner.run(target_path=FIXTURES_DIR)
+            assert len(report.llm_reviews) == 0
+
+    def test_json_report_is_valid(self):
+        """Verify generated JSON report parses correctly and has structure."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            config = _make_test_config(tmpdir)
+            runner = ThreatModelRunner(config)
+            runner.run(target_path=FIXTURES_DIR)
+            json_path = os.path.join(tmpdir, "threat_model_report.json")
+            with open(json_path) as f:
+                data = json.load(f)
+            assert "project_name" in data
+            assert "summary" in data
+            assert "scan_results" in data