diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index 92b2fe9..7b78a4e 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -14,7 +14,6 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS from mvt.common.help import ( HELP_MSG_ANDROID_BACKUP_PASSWORD, HELP_MSG_CHECK_ADB_REMOVED, @@ -40,17 +39,12 @@ from mvt.common.help import ( HELP_MSG_VERSION, HELP_MSG_VIRUS_TOTAL, ) -from mvt.common.logo import logo -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import init_logging, set_verbose_logging -from .cmd_check_androidqf import CmdAndroidCheckAndroidQF -from .cmd_check_backup import CmdAndroidCheckBackup -from .cmd_check_bugreport import CmdAndroidCheckBugreport -from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs -from .command_modules import ANDROID_CHECK_IOCS_MODULES -from .modules.backup.helpers import cli_load_android_backup_password +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-android, including by shell completion on +# every keystroke, so importing it must do no more than build the command tree: +# the forensic modules, the backup parsers and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -74,6 +68,8 @@ def _get_verbose(ctx): def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -101,6 +97,9 @@ def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.obj["disable_indicator_check"] = disable_indicator_update_check ctx.obj["verbose"] = verbose set_verbose_logging(verbose) + + from mvt.common.logo import logo + logo( disable_version_check=disable_update_check, disable_indicator_check=disable_indicator_update_check, @@ -164,6 +163,8 @@ def check_bugreport( verbose, bugreport_path, ): + from .cmd_check_bugreport import CmdAndroidCheckBugreport + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as bug reports are small. @@ -233,6 +234,9 @@ def check_backup( verbose, backup_path, ): + from .cmd_check_backup import CmdAndroidCheckBackup + from .modules.backup.helpers import cli_load_android_backup_password + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) @@ -311,6 +315,9 @@ def check_androidqf( verbose, androidqf_path, ): + from .cmd_check_androidqf import CmdAndroidCheckAndroidQF + from .modules.backup.helpers import cli_load_android_backup_password + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) @@ -393,6 +400,8 @@ def check_intrusion_logs( verbose, logs_path, ): + from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) @@ -446,6 +455,10 @@ def check_intrusion_logs( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import ANDROID_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -472,6 +485,8 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_indicators(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() diff --git a/src/mvt/common/cli_plugins.py b/src/mvt/common/cli_plugins.py index 57f7181..17766ac 100644 --- a/src/mvt/common/cli_plugins.py +++ b/src/mvt/common/cli_plugins.py @@ -15,7 +15,9 @@ from typing import Iterable import click -from .module_loader import CUSTOM_COMMAND_MODULE_PREFIX +# This module is imported by every CLI at start-up, and by shell completion on +# every keystroke, so it must stay cheap: nothing here may import the module +# machinery (mvt.common.module_loader and what it pulls in). IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins" ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins" @@ -24,6 +26,9 @@ NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins" MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS" MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS" MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS" +# Prefix of the import name given to a command file loaded from a path. Shared +# with module_loader, which recognises such files when naming their loggers. +CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" log = logging.getLogger(__name__) diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index d0028b8..51b3836 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -18,6 +18,7 @@ from pathlib import Path from types import ModuleType from typing import Iterable, Optional +from .cli_plugins import CUSTOM_COMMAND_MODULE_PREFIX from .module import MVTModule from .version import MVT_VERSION @@ -27,8 +28,6 @@ EXTERNAL_LOGGER_NAMESPACE = "mvt.ext" PLUGIN_PACKAGE_PREFIX = "mvt_plugin_" _ORIGIN_ATTRIBUTE = "_mvt_module_origin" _PATH_MODULE_PREFIX = "_mvt_custom_module_" -# Shared with cli_plugins, which names a loaded command file this way. -CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" _LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$") log = logging.getLogger(__name__) diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index b71c4ab..2a3e39a 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -14,7 +14,6 @@ from dataclasses import asdict, is_dataclass from typing import Any, Iterator, Union from .log import MVTLogHandler -from mvt.common.config import settings class CustomJSONEncoder(json.JSONEncoder): @@ -272,6 +271,10 @@ def set_verbose_logging(verbose: bool = False): def exec_or_profile(module, globals, locals): """Hook for profiling MVT modules""" + # Imported here so that the CLI modules, which import this one at start-up, + # do not load the settings (and pydantic) before a command runs. + from .config import settings + if settings.PROFILE: cProfile.runctx(module, globals, locals) else: diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 8251fd9..edf7724 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -14,10 +14,7 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.logo import logo from mvt.common.options import MutuallyExclusiveOption -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import ( generate_hashes_from_path, init_logging, @@ -47,13 +44,12 @@ from mvt.common.help import ( HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from mvt.common.password import prompt_password -from .cmd_check_backup import CmdIOSCheckBackup -from .cmd_check_fs import CmdIOSCheckFS -from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose -from .decrypt import DecryptBackup -from .command_modules import IOS_CHECK_IOCS_MODULES + +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-ios, including by shell completion on every +# keystroke, so importing it must do no more than build the command tree: the +# forensic modules, the backup decryption and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -79,6 +75,8 @@ def _get_verbose(ctx): def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -106,6 +104,9 @@ def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.obj["disable_indicator_check"] = disable_indicator_update_check ctx.obj["verbose"] = verbose set_verbose_logging(verbose) + + from mvt.common.logo import logo + logo( disable_version_check=disable_update_check, disable_indicator_check=disable_indicator_update_check, @@ -146,6 +147,8 @@ def version(): @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path, destination) if key_file: @@ -209,6 +212,8 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): ) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) def extract_key(password, key_file, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path) if password: @@ -276,6 +281,8 @@ def check_backup( verbose, backup_path, ): + from .cmd_check_backup import CmdIOSCheckBackup + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -345,6 +352,8 @@ def check_fs( verbose, dump_path, ): + from .cmd_check_fs import CmdIOSCheckFS + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -413,6 +422,8 @@ def check_sysdiagnose( verbose, sysdiagnose_path, ): + from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckSysdiagnose( @@ -467,6 +478,10 @@ def check_sysdiagnose( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import IOS_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -493,6 +508,8 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_iocs(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() diff --git a/tests/test_cli_startup.py b/tests/test_cli_startup.py new file mode 100644 index 0000000..e009256 --- /dev/null +++ b/tests/test_cli_startup.py @@ -0,0 +1,28 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import pytest + +from .plugin_fixtures import run_isolated_python + +# Importing a platform CLI must only build its command tree: the console +# scripts import it before Click can answer a shell completion request, which +# the completion scripts make on every keystroke. Every command imports what +# it runs when it is invoked. Each of these costs tens of milliseconds to +# import and is the sign that a command implementation is imported too early. +HEAVY_MODULES = ("pydantic", "requests", "Crypto", "mvt.common.module") + + +@pytest.mark.parametrize("cli_module", ("mvt.ios.cli", "mvt.android.cli")) +def test_importing_a_cli_does_not_import_the_module_machinery(cli_module, tmp_path): + result = run_isolated_python( + "import sys\n" + f"import {cli_module}\n" + f"print(','.join(name for name in {HEAVY_MODULES!r} if name in sys.modules))\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "", f"{cli_module} imported {result.stdout.strip()}"