Load installed module packages via entry points (#883)

* Load installed module packages via entry points

Python packages can already register custom CLI commands which load
automatically, but custom modules still require --load-module or the
MVT_CUSTOM_MODULES environment variable on every invocation.

Add an mvt.modules entry-point group so installed packages can register
forensic modules which load automatically into every module-running
check-* command. An entry point resolves to an iterable of MVTModule
subclasses, or a callable returning one. Broken entry points are
skipped with a warning so a faulty package cannot break MVT.

* Record the source of loaded modules for auditability

Now that installed module packages load automatically, record where every
module came from:

- --list-modules groups the available modules by source, one line per
  source with the modules comma-separated: MVT itself with its version,
  each installed package with its version and VCS commit when recorded
  (PEP 610 direct_url.json), and each --load-module/MVT_CUSTOM_MODULES
  file with its SHA-256 hash.
- Commands log one line per module source with its version or hash and
  the modules loaded from it, so command.log records exactly which
  modules ran and where they came from.
- Make init_logging() idempotent: a loaded module package importing an
  MVT CLI module would previously add a second console handler and
  duplicate every console log line.

* Route loaded module logging under the mvt.ext namespace

Modules loaded from installed packages or file paths live outside the
mvt logger hierarchy, so their log records never reach MVT's console
and file handlers and instead fall through to logging.lastResort:
alerts print as bare unformatted lines and INFO messages are dropped
entirely.

Add get_module_logger() and use it everywhere module loggers are
created. Built-in mvt.* modules keep their existing logger names, and
everything external is parented under a dedicated mvt.ext namespace so
records reach the handlers and external names can never collide with
MVT's internal logger tree. File-path modules are named after their
file (mvt.ext.<stem>) instead of the mangled internal import name.

Document a naming convention for community module packages:
distribute as mvt-plugin-<name> with import package mvt_plugin_<name>,
including the publishing organization in the name. The prefix is
advisory (loading is by entry point, and it is no mark of
authenticity), but conforming packages get a cleaner logger namespace:
the mvt_plugin_ prefix is stripped, so mvt_plugin_amnesty_custom logs
as mvt.ext.amnesty_custom.

tests/common/test_module_loader.py

@@ -3,10 +3,12 @@ import pytest
 from mvt.common.module import MVTModule
 from mvt.common.module_loader import (
     CustomModuleLoadError,
+    get_module_logger,
     load_custom_modules,
     load_custom_modules_from_path,
     module_supports_command,
 )
+from mvt.ios.modules.mixed.whatsapp import Whatsapp
 
 
 MODULE_TEMPLATE = """
@@ -144,3 +146,44 @@ def test_module_supports_command_honors_supported_commands(tmp_path):
 
     assert module_supports_command(module, "ios", "check-backup")
     assert not module_supports_command(module, "ios", "check-fs")
+
+
+def test_get_module_logger_keeps_builtin_names():
+    assert get_module_logger(Whatsapp).name == "mvt.ios.modules.mixed.whatsapp"
+
+
+def test_get_module_logger_parents_package_modules_under_mvt_ext():
+    class PackageModule(MVTModule):
+        pass
+
+    PackageModule.__module__ = "some_plugin_package.ios.custom"
+
+    assert (
+        get_module_logger(PackageModule).name
+        == "mvt.ext.some_plugin_package.ios.custom"
+    )
+
+
+def test_get_module_logger_strips_the_plugin_package_prefix():
+    class PluginModule(MVTModule):
+        pass
+
+    PluginModule.__module__ = "mvt_plugin_amnesty_custom.ios.custom"
+
+    assert get_module_logger(PluginModule).name == "mvt.ext.amnesty_custom.ios.custom"
+
+
+def test_get_module_logger_only_strips_the_prefix_from_the_top_level():
+    class NestedModule(MVTModule):
+        pass
+
+    NestedModule.__module__ = "other_package.mvt_plugin_sub"
+
+    assert get_module_logger(NestedModule).name == "mvt.ext.other_package.mvt_plugin_sub"
+
+
+def test_get_module_logger_names_path_modules_after_their_file(tmp_path):
+    module_path = _write_module(tmp_path / "my_custom_module.py", "PathModule")
+    module = load_custom_modules_from_path(str(module_path))[0]
+
+    assert get_module_logger(module).name == "mvt.ext.my_custom_module"

tests/common/test_utils.py

@@ -8,6 +8,7 @@ import logging
 import os
 from datetime import datetime
 
+from mvt.common.log import MVTLogHandler
 from mvt.common.utils import (
     CustomJSONEncoder,
     convert_datetime_to_iso,
@@ -16,6 +17,7 @@ from mvt.common.utils import (
     convert_unix_to_utc_datetime,
     generate_hashes_from_path,
     get_sha256_from_file_path,
+    init_logging,
 )
 
 from ..utils import get_artifact_folder
@@ -103,3 +105,20 @@ class TestCustomJSONEncoder:
             json.dumps({"name": "家".encode()}, cls=CustomJSONEncoder)
             == '{"name": "\\u5bb6"}'
         )
+
+
+class TestInitLogging:
+    def test__init_logging_is_idempotent(self):
+        # Loaded module packages may import an MVT CLI module, which calls
+        # init_logging() again at import time. A second call must not add
+        # a duplicate console handler.
+        log = logging.getLogger("mvt")
+        init_logging()
+        handler_count = sum(
+            isinstance(handler, MVTLogHandler) for handler in log.handlers
+        )
+        init_logging()
+        assert (
+            sum(isinstance(handler, MVTLogHandler) for handler in log.handlers)
+            == handler_count
+        )

tests/test_custom_modules.py

@@ -1,3 +1,7 @@
+import hashlib
+import importlib.metadata
+import json
+
 from click.testing import CliRunner
 
 from mvt.android.cli import check_bugreport
@@ -5,7 +9,9 @@ from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF
 from mvt.android.cmd_check_backup import CmdAndroidCheckBackup
 from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport
 from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
+from mvt.common import module_loader
 from mvt.common.module import MVTModule
+from mvt.common.version import MVT_VERSION
 from mvt.ios.cli import check_backup, check_fs
 
 
@@ -106,6 +112,150 @@ def test_custom_modules_load_from_environment_without_cli_flag(tmp_path, monkeyp
     assert "EnvBugreportModule" in result.output
 
 
+class InstalledPackageModule(MVTModule):
+    supported_commands = (("ios", "check-backup"),)
+
+
+def get_installed_package_modules():
+    return [InstalledPackageModule]
+
+
+def _fake_entry_points(monkeypatch, value, name="test-modules"):
+    entry_point = importlib.metadata.EntryPoint(
+        name=name, value=value, group=module_loader.MODULES_ENTRY_POINT_GROUP
+    )
+
+    def fake_entry_points(*, group):
+        assert group == module_loader.MODULES_ENTRY_POINT_GROUP
+        return [entry_point]
+
+    monkeypatch.setattr(
+        module_loader.importlib.metadata, "entry_points", fake_entry_points
+    )
+
+
+def test_installed_module_package_loads_from_entry_point(monkeypatch):
+    _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules")
+
+    modules = module_loader.load_custom_modules()
+
+    assert modules == [InstalledPackageModule]
+
+
+def test_broken_module_entry_point_is_skipped(monkeypatch, caplog):
+    _fake_entry_points(monkeypatch, "nonexistent_module_xyz:get_modules")
+
+    with caplog.at_level("WARNING"):
+        modules = module_loader.load_custom_modules()
+
+    assert modules == []
+    assert "Unable to load modules from entry point" in caplog.text
+
+
+def test_entry_point_module_deduplicated_against_paths(monkeypatch, tmp_path):
+    _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules")
+    module_path = _write_custom_module(
+        tmp_path / "custom.py",
+        "PathLoadedModule",
+        (("ios", "check-backup"),),
+    )
+
+    modules = module_loader.load_custom_modules([str(module_path)])
+
+    assert [module.__name__ for module in modules] == [
+        "InstalledPackageModule",
+        "PathLoadedModule",
+    ]
+
+
+def test_list_modules_shows_module_sources(tmp_path, caplog):
+    module_path = _write_custom_module(
+        tmp_path / "custom.py",
+        "SourcedBackupModule",
+        (("ios", "check-backup"),),
+    )
+    file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest()
+    custom_modules = module_loader.load_custom_modules([str(module_path)])
+
+    from mvt.ios.cmd_check_backup import CmdIOSCheckBackup
+
+    cmd = CmdIOSCheckBackup(target_path=str(tmp_path), custom_modules=custom_modules)
+    cmd.list_modules()
+
+    assert f" - Modules from 'mvt@{MVT_VERSION}':" in caplog.text
+    assert (
+        f" - Modules from '{module_path}' (sha256: {file_sha256}): SourcedBackupModule"
+        in caplog.text
+    )
+
+
+def test_builtin_module_origin():
+    from mvt.ios.modules.backup import BACKUP_MODULES
+
+    origin = module_loader.get_module_origin(BACKUP_MODULES[0])
+
+    assert origin.kind == "builtin"
+    assert origin.name == "mvt"
+    assert origin.version == MVT_VERSION
+
+
+def test_installed_module_origin(monkeypatch):
+    _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules")
+
+    modules = module_loader.load_custom_modules()
+
+    origin = module_loader.get_module_origin(modules[0])
+    assert origin.kind == "package"
+    assert origin.name == "test-modules"
+
+
+def test_distribution_commit_read_from_direct_url():
+    class FakeDistribution:
+        def read_text(self, filename):
+            assert filename == "direct_url.json"
+            return json.dumps(
+                {
+                    "url": "https://github.com/example/example-modules",
+                    "vcs_info": {"commit_id": "abc1234", "vcs": "git"},
+                }
+            )
+
+    assert module_loader._distribution_commit(FakeDistribution()) == "abc1234"
+
+
+def test_command_log_records_loaded_modules(tmp_path):
+    (tmp_path / "Manifest.db").touch()
+    (tmp_path / "Info.plist").touch()
+    module_path = _write_custom_module(
+        tmp_path / "custom.py",
+        "AuditedRunModule",
+        (("ios", "check-backup"),),
+        slug="audited_run_module",
+    )
+    file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest()
+    output_path = tmp_path / "out"
+
+    result = CliRunner().invoke(
+        check_backup,
+        [
+            "--module",
+            "AuditedRunModule",
+            "--load-module",
+            str(module_path),
+            "--output",
+            str(output_path),
+            str(tmp_path),
+        ],
+    )
+
+    assert result.exit_code == 0
+    command_log = (output_path / "command.log").read_text(encoding="utf-8")
+    assert (
+        f"Loaded 1 check-backup modules from '{module_path}' "
+        f"(sha256: {file_sha256}): AuditedRunModule" in command_log
+    )
+
+
 class NestedBugreportModule(MVTModule):
     supported_commands = (("android", "check-bugreport"),)