Merge branch 'main' into fix/bugreport-parser-coverage

src/mvt/android/cli.py

@@ -14,7 +14,6 @@ from mvt.common.cli_plugins import (
     load_cli_commands_option,
     register_cli_plugins,
 )
-from mvt.common.cmd_check_iocs import CmdCheckIOCS
 from mvt.common.help import (
     HELP_MSG_ANDROID_BACKUP_PASSWORD,
     HELP_MSG_CHECK_ADB_REMOVED,
@@ -40,17 +39,12 @@ from mvt.common.help import (
     HELP_MSG_VERSION,
     HELP_MSG_VIRUS_TOTAL,
 )
-from mvt.common.logo import logo
-from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
-from mvt.common.updates import IndicatorsUpdates
 from mvt.common.utils import init_logging, set_verbose_logging
 
-from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
-from .cmd_check_backup import CmdAndroidCheckBackup
-from .cmd_check_bugreport import CmdAndroidCheckBugreport
-from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
-from .command_modules import ANDROID_CHECK_IOCS_MODULES
-from .modules.backup.helpers import cli_load_android_backup_password
+# The commands import what they run only when they are invoked. This module is
+# imported at every start of mvt-android, including by shell completion on
+# every keystroke, so importing it must do no more than build the command tree:
+# the forensic modules, the backup parsers and the update checks stay out of it.
 
 init_logging()
 log = logging.getLogger("mvt")
@@ -74,6 +68,8 @@ def _get_verbose(ctx):
 
 
 def _load_custom_modules(load_module):
+    from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
+
     try:
         return load_custom_modules(load_module)
     except CustomModuleLoadError as exc:
@@ -101,6 +97,9 @@ def cli(ctx, disable_update_check, disable_indicator_update_check, verbose):
     ctx.obj["disable_indicator_check"] = disable_indicator_update_check
     ctx.obj["verbose"] = verbose
     set_verbose_logging(verbose)
+
+    from mvt.common.logo import logo
+
     logo(
         disable_version_check=disable_update_check,
         disable_indicator_check=disable_indicator_update_check,
@@ -164,6 +163,8 @@ def check_bugreport(
     verbose,
     bugreport_path,
 ):
+    from .cmd_check_bugreport import CmdAndroidCheckBugreport
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     custom_modules = _load_custom_modules(load_module)
     # Always generate hashes as bug reports are small.
@@ -233,6 +234,9 @@ def check_backup(
     verbose,
     backup_path,
 ):
+    from .cmd_check_backup import CmdAndroidCheckBackup
+    from .modules.backup.helpers import cli_load_android_backup_password
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     custom_modules = _load_custom_modules(load_module)
 
@@ -311,6 +315,9 @@ def check_androidqf(
     verbose,
     androidqf_path,
 ):
+    from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
+    from .modules.backup.helpers import cli_load_android_backup_password
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     custom_modules = _load_custom_modules(load_module)
 
@@ -393,6 +400,8 @@ def check_intrusion_logs(
     verbose,
     logs_path,
 ):
+    from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     custom_modules = _load_custom_modules(load_module)
 
@@ -446,6 +455,10 @@ def check_intrusion_logs(
 @click.argument("FOLDER", type=click.Path(exists=True))
 @click.pass_context
 def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
+    from mvt.common.cmd_check_iocs import CmdCheckIOCS
+
+    from .command_modules import ANDROID_CHECK_IOCS_MODULES
+
     custom_modules = _load_custom_modules(load_module)
     cmd = CmdCheckIOCS(
         target_path=folder,
@@ -472,6 +485,8 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
 # ==============================================================================
 @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
 def download_indicators():
+    from mvt.common.updates import IndicatorsUpdates
+
     ioc_updates = IndicatorsUpdates()
     ioc_updates.update()
 

src/mvt/common/cli_plugins.py

@@ -15,7 +15,9 @@ from typing import Iterable
 
 import click
 
-from .module_loader import CUSTOM_COMMAND_MODULE_PREFIX
+# This module is imported by every CLI at start-up, and by shell completion on
+# every keystroke, so it must stay cheap: nothing here may import the module
+# machinery (mvt.common.module_loader and what it pulls in).
 
 IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins"
 ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins"
@@ -24,6 +26,9 @@ NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins"
 MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS"
 MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS"
 MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS"
+# Prefix of the import name given to a command file loaded from a path. Shared
+# with module_loader, which recognises such files when naming their loggers.
+CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_"
 
 log = logging.getLogger(__name__)
 

src/mvt/common/module_loader.py

@@ -18,6 +18,7 @@ from pathlib import Path
 from types import ModuleType
 from typing import Iterable, Optional
 
+from .cli_plugins import CUSTOM_COMMAND_MODULE_PREFIX
 from .module import MVTModule
 from .version import MVT_VERSION
 
@@ -27,8 +28,6 @@ EXTERNAL_LOGGER_NAMESPACE = "mvt.ext"
 PLUGIN_PACKAGE_PREFIX = "mvt_plugin_"
 _ORIGIN_ATTRIBUTE = "_mvt_module_origin"
 _PATH_MODULE_PREFIX = "_mvt_custom_module_"
-# Shared with cli_plugins, which names a loaded command file this way.
-CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_"
 _LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$")
 log = logging.getLogger(__name__)
 

src/mvt/common/utils.py

@@ -14,7 +14,6 @@ from dataclasses import asdict, is_dataclass
 from typing import Any, Iterator, Union
 
 from .log import MVTLogHandler
-from mvt.common.config import settings
 
 
 class CustomJSONEncoder(json.JSONEncoder):
@@ -272,6 +271,10 @@ def set_verbose_logging(verbose: bool = False):
 
 def exec_or_profile(module, globals, locals):
     """Hook for profiling MVT modules"""
+    # Imported here so that the CLI modules, which import this one at start-up,
+    # do not load the settings (and pydantic) before a command runs.
+    from .config import settings
+
     if settings.PROFILE:
         cProfile.runctx(module, globals, locals)
     else:

src/mvt/ios/cli.py

@@ -14,10 +14,7 @@ from mvt.common.cli_plugins import (
     load_cli_commands_option,
     register_cli_plugins,
 )
-from mvt.common.cmd_check_iocs import CmdCheckIOCS
-from mvt.common.logo import logo
 from mvt.common.options import MutuallyExclusiveOption
-from mvt.common.updates import IndicatorsUpdates
 from mvt.common.utils import (
     generate_hashes_from_path,
     init_logging,
@@ -47,13 +44,12 @@ from mvt.common.help import (
     HELP_MSG_DISABLE_UPDATE_CHECK,
     HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
 )
-from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
 from mvt.common.password import prompt_password
-from .cmd_check_backup import CmdIOSCheckBackup
-from .cmd_check_fs import CmdIOSCheckFS
-from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
-from .decrypt import DecryptBackup
-from .command_modules import IOS_CHECK_IOCS_MODULES
+
+# The commands import what they run only when they are invoked. This module is
+# imported at every start of mvt-ios, including by shell completion on every
+# keystroke, so importing it must do no more than build the command tree: the
+# forensic modules, the backup decryption and the update checks stay out of it.
 
 init_logging()
 log = logging.getLogger("mvt")
@@ -79,6 +75,8 @@ def _get_verbose(ctx):
 
 
 def _load_custom_modules(load_module):
+    from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
+
     try:
         return load_custom_modules(load_module)
     except CustomModuleLoadError as exc:
@@ -106,6 +104,9 @@ def cli(ctx, disable_update_check, disable_indicator_update_check, verbose):
     ctx.obj["disable_indicator_check"] = disable_indicator_update_check
     ctx.obj["verbose"] = verbose
     set_verbose_logging(verbose)
+
+    from mvt.common.logo import logo
+
     logo(
         disable_version_check=disable_update_check,
         disable_indicator_check=disable_indicator_update_check,
@@ -146,6 +147,8 @@ def version():
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
 def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
+    from .decrypt import DecryptBackup
+
     backup = DecryptBackup(backup_path, destination)
 
     if key_file:
@@ -209,6 +212,8 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
 )
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 def extract_key(password, key_file, backup_path):
+    from .decrypt import DecryptBackup
+
     backup = DecryptBackup(backup_path)
 
     if password:
@@ -276,6 +281,8 @@ def check_backup(
     verbose,
     backup_path,
 ):
+    from .cmd_check_backup import CmdIOSCheckBackup
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     module_options = {"fast_mode": fast}
     custom_modules = _load_custom_modules(load_module)
@@ -345,6 +352,8 @@ def check_fs(
     verbose,
     dump_path,
 ):
+    from .cmd_check_fs import CmdIOSCheckFS
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     module_options = {"fast_mode": fast}
     custom_modules = _load_custom_modules(load_module)
@@ -413,6 +422,8 @@ def check_sysdiagnose(
     verbose,
     sysdiagnose_path,
 ):
+    from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
+
     set_verbose_logging(verbose or _get_verbose(ctx))
     custom_modules = _load_custom_modules(load_module)
     cmd = CmdIOSCheckSysdiagnose(
@@ -467,6 +478,10 @@ def check_sysdiagnose(
 @click.argument("FOLDER", type=click.Path(exists=True))
 @click.pass_context
 def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
+    from mvt.common.cmd_check_iocs import CmdCheckIOCS
+
+    from .command_modules import IOS_CHECK_IOCS_MODULES
+
     custom_modules = _load_custom_modules(load_module)
     cmd = CmdCheckIOCS(
         target_path=folder,
@@ -493,6 +508,8 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
 # ==============================================================================
 @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
 def download_iocs():
+    from mvt.common.updates import IndicatorsUpdates
+
     ioc_updates = IndicatorsUpdates()
     ioc_updates.update()
 

tests/test_cli_startup.py

@@ -0,0 +1,28 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2026 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import pytest
+
+from .plugin_fixtures import run_isolated_python
+
+# Importing a platform CLI must only build its command tree: the console
+# scripts import it before Click can answer a shell completion request, which
+# the completion scripts make on every keystroke. Every command imports what
+# it runs when it is invoked. Each of these costs tens of milliseconds to
+# import and is the sign that a command implementation is imported too early.
+HEAVY_MODULES = ("pydantic", "requests", "Crypto", "mvt.common.module")
+
+
+@pytest.mark.parametrize("cli_module", ("mvt.ios.cli", "mvt.android.cli"))
+def test_importing_a_cli_does_not_import_the_module_machinery(cli_module, tmp_path):
+    result = run_isolated_python(
+        "import sys\n"
+        f"import {cli_module}\n"
+        f"print(','.join(name for name in {HEAVY_MODULES!r} if name in sys.modules))\n",
+        home=tmp_path / "home",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert result.stdout.strip() == "", f"{cli_module} imported {result.stdout.strip()}"