From 638937e838e9707c73cfafa76bc383226616d1e7 Mon Sep 17 00:00:00 2001 From: besendorf Date: Wed, 24 Jun 2026 12:41:00 +0200 Subject: [PATCH 1/4] Handle malformed AndroidQF backups (#824) --- src/mvt/android/cmd_check_androidqf.py | 11 +++++++++-- src/mvt/android/cmd_check_backup.py | 23 ++++++++++++++++++++++- src/mvt/android/modules/androidqf/sms.py | 4 ++-- src/mvt/android/parsers/backup.py | 17 ++++++++++------- tests/android/test_backup_parser.py | 14 +++++++++++++- tests/test_check_android_androidqf.py | 17 +++++++++++++++++ 6 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/mvt/android/cmd_check_androidqf.py b/src/mvt/android/cmd_check_androidqf.py index 54dbc3d..04c981b 100644 --- a/src/mvt/android/cmd_check_androidqf.py +++ b/src/mvt/android/cmd_check_androidqf.py @@ -13,7 +13,7 @@ from typing import List, Optional from mvt.android.artifacts.getprop import GetProp from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs -from mvt.android.cmd_check_backup import CmdAndroidCheckBackup +from mvt.android.cmd_check_backup import CmdAndroidCheckBackup, InvalidAndroidBackup from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport from mvt.common.command import Command from mvt.common.indicators import Indicators @@ -240,7 +240,14 @@ class CmdAndroidCheckAndroidQF(Command): hashes=self.hashes, sub_command=True, ) - cmd.from_ab(backup) + try: + cmd.from_ab(backup) + except InvalidAndroidBackup as exc: + self.log.warning( + "Skipping backup modules as backup.ab is malformed: %s", exc + ) + return False + cmd.run() self.timeline.extend(cmd.timeline) diff --git a/src/mvt/android/cmd_check_backup.py b/src/mvt/android/cmd_check_backup.py index 1b239f1..3f91f07 100644 --- a/src/mvt/android/cmd_check_backup.py +++ b/src/mvt/android/cmd_check_backup.py @@ -27,6 +27,10 @@ from .modules.backup import BACKUP_MODULES log = logging.getLogger(__name__) +class InvalidAndroidBackup(Exception): + pass + + class CmdAndroidCheckBackup(Command): def __init__( self, @@ -68,6 +72,10 @@ class CmdAndroidCheckBackup(Command): self.__type = "ab" header = parse_ab_header(ab_file_bytes) if not header["backup"]: + if self.sub_command: + raise InvalidAndroidBackup( + "Invalid backup format, file should be in .ab format" + ) log.critical("Invalid backup format, file should be in .ab format") sys.exit(1) @@ -83,12 +91,25 @@ class CmdAndroidCheckBackup(Command): log.critical("Invalid backup password") sys.exit(1) except AndroidBackupParsingError as exc: + if self.sub_command: + raise InvalidAndroidBackup( + f"Impossible to parse this backup file: {exc}" + ) from exc log.critical("Impossible to parse this backup file: %s", exc) log.critical("Please use Android Backup Extractor (ABE) instead") sys.exit(1) dbytes = io.BytesIO(tardata) - self.__tar = tarfile.open(fileobj=dbytes) + try: + self.__tar = tarfile.open(fileobj=dbytes) + except tarfile.TarError as exc: + if self.sub_command: + raise InvalidAndroidBackup( + f"Impossible to parse this backup file: {exc}" + ) from exc + log.critical("Impossible to parse this backup file: %s", exc) + log.critical("Please use Android Backup Extractor (ABE) instead") + sys.exit(1) for member in self.__tar: self.__files.append(member.name) diff --git a/src/mvt/android/modules/androidqf/sms.py b/src/mvt/android/modules/androidqf/sms.py index bcbd226..64aaab6 100644 --- a/src/mvt/android/modules/androidqf/sms.py +++ b/src/mvt/android/modules/androidqf/sms.py @@ -58,7 +58,7 @@ class SMS(AndroidQFModule): def parse_backup(self, data): header = parse_ab_header(data) if not header["backup"]: - self.log.critical("Invalid backup format, backup.ab was not analysed") + self.log.warning("Invalid backup format, backup.ab was not analysed") return password = None @@ -76,7 +76,7 @@ class SMS(AndroidQFModule): self.log.critical("Invalid backup password") return except AndroidBackupParsingError: - self.log.critical( + self.log.warning( "Impossible to parse this backup file, please use" " Android Backup Extractor instead" ) diff --git a/src/mvt/android/parsers/backup.py b/src/mvt/android/parsers/backup.py index c81ecd0..8c9bb5c 100644 --- a/src/mvt/android/parsers/backup.py +++ b/src/mvt/android/parsers/backup.py @@ -48,13 +48,16 @@ def parse_ab_header(data): 'encryption': "none", 'version': 4} """ if data.startswith(b"ANDROID BACKUP"): - [_, version, is_compressed, encryption, _] = data.split(b"\n", 4) - return { - "backup": True, - "compression": (is_compressed == b"1"), - "version": int(version), - "encryption": encryption.decode("utf-8"), - } + try: + [_, version, is_compressed, encryption, _] = data.split(b"\n", 4) + return { + "backup": True, + "compression": (is_compressed == b"1"), + "version": int(version), + "encryption": encryption.decode("utf-8"), + } + except (UnicodeDecodeError, ValueError): + pass return {"backup": False, "compression": None, "version": None, "encryption": None} diff --git a/tests/android/test_backup_parser.py b/tests/android/test_backup_parser.py index 5bd5b99..6e5be72 100644 --- a/tests/android/test_backup_parser.py +++ b/tests/android/test_backup_parser.py @@ -5,12 +5,24 @@ import hashlib -from mvt.android.parsers.backup import parse_backup_file, parse_tar_for_sms +from mvt.android.parsers.backup import ( + parse_ab_header, + parse_backup_file, + parse_tar_for_sms, +) from ..utils import get_artifact class TestBackupParsing: + def test_parse_incomplete_header(self): + assert parse_ab_header(b"ANDROID BACKUP\n") == { + "backup": False, + "compression": None, + "version": None, + "encryption": None, + } + def test_parsing_noencryption(self): file = get_artifact("android_backup/backup.ab") with open(file, "rb") as f: diff --git a/tests/test_check_android_androidqf.py b/tests/test_check_android_androidqf.py index c6e4221..c4c3410 100644 --- a/tests/test_check_android_androidqf.py +++ b/tests/test_check_android_androidqf.py @@ -3,7 +3,9 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import logging import os +import shutil from click.testing import CliRunner @@ -68,3 +70,18 @@ class TestCheckAndroidqfCommand: assert result.exit_code == 0 del os.environ["MVT_ANDROID_BACKUP_PASSWORD"] settings.__init__() # Reset settings + + def test_check_malformed_backup_skips_backup_modules(self, tmp_path, caplog): + path = tmp_path / "androidqf" + shutil.copytree(os.path.join(get_artifact_folder(), "androidqf"), path) + (path / "backup.ab").write_bytes(b"") + + runner = CliRunner() + with caplog.at_level(logging.WARNING): + result = runner.invoke(check_androidqf, [str(path)]) + + assert result.exit_code == 0 + assert "Skipping backup modules as backup.ab is malformed" in caplog.text + assert not any( + record.levelname in {"CRITICAL", "FATAL"} for record in caplog.records + ) From 2689176c0eff33d56fabe0050470ac250ada088d Mon Sep 17 00:00:00 2001 From: besendorf Date: Wed, 1 Jul 2026 12:33:38 +0200 Subject: [PATCH 2/4] Add custom module loading (#816) --- README.md | 5 + docs/development.md | 76 ++++++++ src/mvt/android/cli.py | 71 ++++++- src/mvt/android/cmd_check_androidqf.py | 7 + src/mvt/android/cmd_check_backup.py | 4 + src/mvt/android/cmd_check_bugreport.py | 4 + src/mvt/android/cmd_check_intrusion_logs.py | 4 + src/mvt/common/cmd_check_iocs.py | 7 +- src/mvt/common/command.py | 28 ++- src/mvt/common/help.py | 4 + src/mvt/common/module.py | 1 + src/mvt/common/module_loader.py | 127 +++++++++++++ src/mvt/ios/cli.py | 63 ++++++- src/mvt/ios/cmd_check_backup.py | 4 + src/mvt/ios/cmd_check_fs.py | 4 + tests/common/test_command.py | 56 ++++++ tests/common/test_module_loader.py | 145 +++++++++++++++ tests/test_custom_modules.py | 193 ++++++++++++++++++++ 18 files changed, 793 insertions(+), 10 deletions(-) create mode 100644 src/mvt/common/module_loader.py create mode 100644 tests/common/test_module_loader.py create mode 100644 tests/test_custom_modules.py diff --git a/README.md b/README.md index 04db0ab..cb651a1 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,11 @@ For alternative installation options and known issues, please refer to the [docu MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +Module-running `check-*` commands can load custom Python modules with +`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the +[development documentation](https://docs.mvt.re/en/latest/development/) for +details. + ## License diff --git a/docs/development.md b/docs/development.md index 6a15114..93a612f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -39,6 +39,82 @@ Selecting a single module also runs its transitive dependencies. If a dependency is unavailable or the dependency graph contains a cycle, the command logs a warning and does not run any modules. +## Custom modules + +Module-running `check-*` commands can load custom modules from Python files that +are not installed as part of MVT. Load one file with: + +```bash +mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup +``` + +You can also load a folder. MVT loads non-hidden top-level `*.py` files in +sorted order and skips `__init__.py`: + +```bash +mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump +``` + +Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This +folder is loaded before any `--load-module` path: + +```bash +MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip +``` + +Custom modules are normal `MVTModule` subclasses: + +```python +from mvt.common.module import MVTModule + + +class ExampleCustomModule(MVTModule): + supported_commands = (("ios", "check-backup"), ("ios", "check-fs")) + slug = "example_custom_module" + + def run(self): + self.results = [{"message": "custom module ran"}] + + def check_indicators(self): + pass + + def serialize(self, result): + return None +``` + +Use `supported_commands` to restrict a module to specific platform/command +pairs. Missing or empty `supported_commands` means the module is available to +all commands, which keeps older modules compatible. Supported pairs are: + +```python +("ios", "check-backup") +("ios", "check-fs") +("ios", "check-iocs") +("android", "check-backup") +("android", "check-bugreport") +("android", "check-androidqf") +("android", "check-intrusion-logs") +("android", "check-iocs") +``` + +Custom modules can depend on existing MVT module classes. Dependencies are +resolved with the same ordering logic as built-in modules, and custom modules +are appended after built-ins before ordering: + +```python +from mvt.common.module import MVTModule +from mvt.ios.modules.backup.manifest import Manifest + + +class DependentCustomModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + dependencies = (Manifest,) + + def run(self): + manifest_results = self.get_dependency_results(Manifest) + self.results = [{"manifest_entries": len(manifest_results)}] +``` + ## Profiling Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index cdb3b3e..364f4ef 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -22,6 +22,7 @@ from mvt.common.help import ( HELP_MSG_HASHES, HELP_MSG_IOC, HELP_MSG_LIST_MODULES, + HELP_MSG_LOAD_MODULE, HELP_MSG_MODULE, HELP_MSG_NONINTERACTIVE, HELP_MSG_OUTPUT, @@ -30,6 +31,7 @@ from mvt.common.help import ( HELP_MSG_VERSION, ) from mvt.common.logo import logo +from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import init_logging, set_verbose_logging @@ -59,6 +61,13 @@ def _get_disable_flags(ctx): ) +def _load_custom_modules(load_module): + try: + return load_custom_modules(load_module) + except CustomModuleLoadError as exc: + raise click.ClickException(str(exc)) from exc + + # ============================================================================== # Main # ============================================================================== @@ -119,11 +128,28 @@ def check_adb(ctx): @click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.argument("BUGREPORT_PATH", type=click.Path(exists=True)) @click.pass_context -def check_bugreport(ctx, iocs, output, list_modules, module, verbose, bugreport_path): +def check_bugreport( + ctx, + iocs, + output, + list_modules, + module, + load_module, + verbose, + bugreport_path, +): set_verbose_logging(verbose) + custom_modules = _load_custom_modules(load_module) # Always generate hashes as bug reports are small. cmd = CmdAndroidCheckBugreport( target_path=bugreport_path, @@ -133,6 +159,7 @@ def check_bugreport(ctx, iocs, output, list_modules, module, verbose, bugreport_ hashes=True, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, ) if list_modules: @@ -164,6 +191,13 @@ def check_bugreport(ctx, iocs, output, list_modules, module, verbose, bugreport_ ) @click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @@ -174,12 +208,14 @@ def check_backup( iocs, output, list_modules, + load_module, non_interactive, backup_password, verbose, backup_path, ): set_verbose_logging(verbose) + custom_modules = _load_custom_modules(load_module) # Always generate hashes as backups are generally small. cmd = CmdAndroidCheckBackup( @@ -193,6 +229,7 @@ def check_backup( }, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, ) if list_modules: @@ -223,6 +260,13 @@ def check_backup( @click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) @@ -235,6 +279,7 @@ def check_androidqf( output, list_modules, module, + load_module, hashes, non_interactive, backup_password, @@ -242,6 +287,7 @@ def check_androidqf( androidqf_path, ): set_verbose_logging(verbose) + custom_modules = _load_custom_modules(load_module) cmd = CmdAndroidCheckAndroidQF( target_path=androidqf_path, @@ -255,6 +301,7 @@ def check_androidqf( }, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, ) if list_modules: @@ -288,6 +335,13 @@ def check_androidqf( @click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.option( "--timezone", "-t", @@ -307,11 +361,13 @@ def check_intrusion_logs( output, list_modules, module, + load_module, timezone, verbose, logs_path, ): set_verbose_logging(verbose) + custom_modules = _load_custom_modules(load_module) module_options = {} if timezone: @@ -325,6 +381,7 @@ def check_intrusion_logs( module_options=module_options if module_options else None, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, ) if list_modules: @@ -352,15 +409,25 @@ def check_intrusion_logs( ) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context -def check_iocs(ctx, iocs, list_modules, module, folder): +def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, ioc_files=iocs, module_name=module, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, + platform="android", ) cmd.modules = ( BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES diff --git a/src/mvt/android/cmd_check_androidqf.py b/src/mvt/android/cmd_check_androidqf.py index 04c981b..41ad802 100644 --- a/src/mvt/android/cmd_check_androidqf.py +++ b/src/mvt/android/cmd_check_androidqf.py @@ -17,6 +17,7 @@ from mvt.android.cmd_check_backup import CmdAndroidCheckBackup, InvalidAndroidBa from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport from mvt.common.command import Command from mvt.common.indicators import Indicators +from mvt.common.module import MVTModule from .modules.androidqf import ANDROIDQF_MODULES from .modules.androidqf.base import AndroidQFModule @@ -50,6 +51,7 @@ class CmdAndroidCheckAndroidQF(Command): sub_command: Optional[bool] = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: super().__init__( target_path=target_path, @@ -64,8 +66,10 @@ class CmdAndroidCheckAndroidQF(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = "android" self.name = "check-androidqf" self.modules = ANDROIDQF_MODULES @@ -210,6 +214,7 @@ class CmdAndroidCheckAndroidQF(Command): module_options=self.module_options, hashes=self.hashes, sub_command=True, + custom_modules=self.custom_modules, ) cmd.from_zip(bugreport) cmd.run() @@ -239,6 +244,7 @@ class CmdAndroidCheckAndroidQF(Command): module_options=self.module_options, hashes=self.hashes, sub_command=True, + custom_modules=self.custom_modules, ) try: cmd.from_ab(backup) @@ -318,6 +324,7 @@ class CmdAndroidCheckAndroidQF(Command): module_options=adv_module_options, hashes=self.hashes, sub_command=True, + custom_modules=self.custom_modules, ) cmd.run() diff --git a/src/mvt/android/cmd_check_backup.py b/src/mvt/android/cmd_check_backup.py index 3f91f07..b75bb34 100644 --- a/src/mvt/android/cmd_check_backup.py +++ b/src/mvt/android/cmd_check_backup.py @@ -21,6 +21,7 @@ from mvt.android.parsers.backup import ( ) from mvt.common.command import Command from mvt.common.indicators import Indicators +from mvt.common.module import MVTModule from .modules.backup import BACKUP_MODULES @@ -45,6 +46,7 @@ class CmdAndroidCheckBackup(Command): sub_command: Optional[bool] = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: super().__init__( target_path=target_path, @@ -59,8 +61,10 @@ class CmdAndroidCheckBackup(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = "android" self.name = "check-backup" self.modules = BACKUP_MODULES diff --git a/src/mvt/android/cmd_check_bugreport.py b/src/mvt/android/cmd_check_bugreport.py index b6f1367..1c03c6d 100644 --- a/src/mvt/android/cmd_check_bugreport.py +++ b/src/mvt/android/cmd_check_bugreport.py @@ -12,6 +12,7 @@ from zipfile import ZipFile from mvt.android.modules.bugreport.base import BugReportModule from mvt.common.command import Command from mvt.common.indicators import Indicators +from mvt.common.module import MVTModule from .modules.bugreport import BUGREPORT_MODULES @@ -32,6 +33,7 @@ class CmdAndroidCheckBugreport(Command): sub_command: Optional[bool] = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: super().__init__( target_path=target_path, @@ -46,8 +48,10 @@ class CmdAndroidCheckBugreport(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = "android" self.name = "check-bugreport" self.modules = BUGREPORT_MODULES diff --git a/src/mvt/android/cmd_check_intrusion_logs.py b/src/mvt/android/cmd_check_intrusion_logs.py index 8541f9a..95f2089 100644 --- a/src/mvt/android/cmd_check_intrusion_logs.py +++ b/src/mvt/android/cmd_check_intrusion_logs.py @@ -9,6 +9,7 @@ from typing import Optional from mvt.common.command import Command from mvt.common.indicators import Indicators +from mvt.common.module import MVTModule from .modules.intrusion_logs import ( INTRUSION_LOGS_MODULES, @@ -35,6 +36,7 @@ class CmdAndroidCheckIntrusionLogs(Command): sub_command: Optional[bool] = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: super().__init__( target_path=target_path, @@ -49,8 +51,10 @@ class CmdAndroidCheckIntrusionLogs(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = "android" self.name = "check-intrusion-logs" self.modules = INTRUSION_LOGS_MODULES self._all_events: dict[str, list[dict]] = {} diff --git a/src/mvt/common/cmd_check_iocs.py b/src/mvt/common/cmd_check_iocs.py index 35700ea..c227659 100644 --- a/src/mvt/common/cmd_check_iocs.py +++ b/src/mvt/common/cmd_check_iocs.py @@ -8,6 +8,7 @@ import os from typing import Optional from mvt.common.command import Command +from mvt.common.module import MVTModule from mvt.common.utils import exec_or_profile log = logging.getLogger(__name__) @@ -26,6 +27,8 @@ class CmdCheckIOCS(Command): sub_command: Optional[bool] = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, + platform: str = "", ) -> None: super().__init__( target_path=target_path, @@ -39,14 +42,16 @@ class CmdCheckIOCS(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = platform self.name = "check-iocs" def run(self) -> None: assert self.target_path is not None all_modules = [] - for entry in self.modules: + for entry in self._available_modules(): if entry not in all_modules: all_modules.append(entry) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index 35daaeb..8d21aa3 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -19,6 +19,7 @@ from .alerts import AlertLevel, AlertStore from .config import settings from .indicators import Indicators from .module import EncryptedBackupError, MVTModule, run_module, save_timeline +from .module_loader import module_supports_command from .module_types import ModuleTimeline from .utils import ( CustomJSONEncoder, @@ -44,9 +45,12 @@ class Command: log: logging.Logger = logging.getLogger(__name__), disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: self.name = "" + self.platform = "" self.modules: list[type[MVTModule]] = [] + self.custom_modules = custom_modules if custom_modules else [] self.target_path = target_path self.results_path = results_path @@ -199,9 +203,24 @@ class Command: def list_modules(self) -> None: self.log.info("Following is the list of available %s modules:", self.name) - for module in self.modules: + for module in self._available_modules(): self.log.info(" - %s", module.__name__) + def _available_modules(self) -> list[type[MVTModule]]: + modules = list(self.modules) + modules.extend( + module + for module in self.custom_modules + if module_supports_command(module, self.platform, self.name) + ) + + deduplicated = [] + for module in modules: + if module not in deduplicated: + deduplicated.append(module) + + return deduplicated + def init(self) -> None: raise NotImplementedError @@ -262,14 +281,15 @@ class Command: def _ordered_modules(self) -> Optional[list[type[MVTModule]]]: """Return enabled modules in stable topological order.""" - module_indexes = {module: index for index, module in enumerate(self.modules)} + modules = self._available_modules() + module_indexes = {module: index for index, module in enumerate(modules)} if self.module_name: selected = [ - module for module in self.modules if module.__name__ == self.module_name + module for module in modules if module.__name__ == self.module_name ] else: - selected = [module for module in self.modules if module.enabled] + selected = [module for module in modules if module.enabled] required = set(selected) pending = list(selected) diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 535a059..4cba131 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -10,6 +10,10 @@ HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)" HELP_MSG_FAST = "Avoid running time/resource consuming features" HELP_MSG_LIST_MODULES = "Print list of available modules and exit" HELP_MSG_MODULE = "Name of a single module you would like to run instead of all" +HELP_MSG_LOAD_MODULE = ( + "Load custom MVT module(s) from a Python file or folder " + "(can be invoked multiple times)" +) HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing" HELP_MSG_HASHES = "Generate hashes of all the files analyzed" HELP_MSG_VERBOSE = "Verbose mode" diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index 6e231be..14c5fe5 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -44,6 +44,7 @@ class MVTModule: enabled: bool = True slug: Optional[str] = None dependencies: Sequence[type["MVTModule"]] = () + supported_commands: Sequence[tuple[str, str]] = () def __init__( self, diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py new file mode 100644 index 0000000..890ed0f --- /dev/null +++ b/src/mvt/common/module_loader.py @@ -0,0 +1,127 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import hashlib +import importlib.util +import inspect +import os +import sys +from pathlib import Path +from types import ModuleType +from typing import Iterable, Optional + +from .module import MVTModule + +MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES" + + +class CustomModuleLoadError(Exception): + pass + + +def _module_name_for_path(path: Path) -> str: + digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] + return f"_mvt_custom_module_{path.stem}_{digest}" + + +def _iter_module_files(path: Path) -> Iterable[Path]: + if path.is_file(): + if path.suffix != ".py": + raise CustomModuleLoadError(f"Custom module file is not a Python file: {path}") + yield path + return + + if path.is_dir(): + for child in sorted(path.iterdir()): + if child.name.startswith("."): + continue + if child.name == "__init__.py": + continue + if child.is_file() and child.suffix == ".py": + yield child + return + + raise CustomModuleLoadError(f"Custom module path does not exist: {path}") + + +def _load_python_file(path: Path) -> ModuleType: + module_name = _module_name_for_path(path) + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise CustomModuleLoadError(f"Unable to load custom module file: {path}") + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception as exc: + raise CustomModuleLoadError(f"Unable to import custom module {path}: {exc}") from exc + + return module + + +def discover_mvt_modules(module: ModuleType) -> list[type[MVTModule]]: + modules = [] + for _, obj in inspect.getmembers(module, inspect.isclass): + if obj is MVTModule: + continue + if obj.__module__ != module.__name__: + continue + if not issubclass(obj, MVTModule): + continue + modules.append(obj) + + return modules + + +def load_custom_modules_from_path(path: str) -> list[type[MVTModule]]: + custom_modules: list[type[MVTModule]] = [] + seen: set[tuple[str, str]] = set() + resolved_path = Path(path).expanduser().resolve() + + for module_file in _iter_module_files(resolved_path): + loaded_module = _load_python_file(module_file) + for module_class in discover_mvt_modules(loaded_module): + key = (str(module_file), module_class.__qualname__) + if key in seen: + continue + seen.add(key) + custom_modules.append(module_class) + + return custom_modules + + +def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTModule]]: + search_paths: list[str] = [] + env_path = os.environ.get(MVT_CUSTOM_MODULES_ENV) + if env_path: + search_paths.append(env_path) + if paths: + search_paths.extend(paths) + + custom_modules: list[type[MVTModule]] = [] + seen: set[tuple[str, str]] = set() + for path in search_paths: + for module_class in load_custom_modules_from_path(path): + source = Path(inspect.getfile(module_class)).resolve() + key = (str(source), module_class.__qualname__) + if key in seen: + continue + seen.add(key) + custom_modules.append(module_class) + + return custom_modules + + +def module_supports_command( + module_class: type[MVTModule], + platform: str, + command: str, +) -> bool: + supported_commands = getattr(module_class, "supported_commands", None) + if not supported_commands: + return True + + return (platform, command) in {tuple(entry) for entry in supported_commands} diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 446a0e6..d9f634c 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -31,6 +31,7 @@ from mvt.common.help import ( HELP_MSG_OUTPUT, HELP_MSG_FAST, HELP_MSG_LIST_MODULES, + HELP_MSG_LOAD_MODULE, HELP_MSG_MODULE, HELP_MSG_VERBOSE, HELP_MSG_CHECK_FS, @@ -40,6 +41,7 @@ from mvt.common.help import ( HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from .cmd_check_backup import CmdIOSCheckBackup from .cmd_check_fs import CmdIOSCheckFS from .decrypt import DecryptBackup @@ -65,6 +67,13 @@ def _get_disable_flags(ctx): ) +def _load_custom_modules(load_module): + try: + return load_custom_modules(load_module) + except CustomModuleLoadError as exc: + raise click.ClickException(str(exc)) from exc + + # ============================================================================== # Main # ============================================================================== @@ -229,15 +238,32 @@ def extract_key(password, key_file, backup_path): @click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( - ctx, iocs, output, fast, list_modules, module, hashes, verbose, backup_path + ctx, + iocs, + output, + fast, + list_modules, + module, + load_module, + hashes, + verbose, + backup_path, ): set_verbose_logging(verbose) module_options = {"fast_mode": fast} + custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckBackup( target_path=backup_path, @@ -248,6 +274,7 @@ def check_backup( hashes=hashes, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, ) if list_modules: @@ -277,13 +304,32 @@ def check_backup( @click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.argument("DUMP_PATH", type=click.Path(exists=True)) @click.pass_context -def check_fs(ctx, iocs, output, fast, list_modules, module, hashes, verbose, dump_path): +def check_fs( + ctx, + iocs, + output, + fast, + list_modules, + module, + load_module, + hashes, + verbose, + dump_path, +): set_verbose_logging(verbose) module_options = {"fast_mode": fast} + custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckFS( target_path=dump_path, @@ -294,6 +340,7 @@ def check_fs(ctx, iocs, output, fast, list_modules, module, hashes, verbose, dum hashes=hashes, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, ) if list_modules: @@ -321,15 +368,25 @@ def check_fs(ctx, iocs, output, fast, list_modules, module, hashes, verbose, dum ) @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES) @click.option("--module", "-m", help=HELP_MSG_MODULE) +@click.option( + "--load-module", + type=click.Path(exists=True), + multiple=True, + default=[], + help=HELP_MSG_LOAD_MODULE, +) @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context -def check_iocs(ctx, iocs, list_modules, module, folder): +def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, ioc_files=iocs, module_name=module, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], + custom_modules=custom_modules, + platform="ios", ) cmd.modules = BACKUP_MODULES + FS_MODULES + MIXED_MODULES diff --git a/src/mvt/ios/cmd_check_backup.py b/src/mvt/ios/cmd_check_backup.py index 9200964..4264a7f 100644 --- a/src/mvt/ios/cmd_check_backup.py +++ b/src/mvt/ios/cmd_check_backup.py @@ -8,6 +8,7 @@ from typing import Optional from mvt.common.command import Command from mvt.common.indicators import Indicators +from mvt.common.module import MVTModule from .modules.backup import BACKUP_MODULES from .modules.mixed import MIXED_MODULES @@ -29,6 +30,7 @@ class CmdIOSCheckBackup(Command): sub_command: bool = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: super().__init__( target_path=target_path, @@ -43,8 +45,10 @@ class CmdIOSCheckBackup(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = "ios" self.name = "check-backup" self.modules = BACKUP_MODULES + MIXED_MODULES diff --git a/src/mvt/ios/cmd_check_fs.py b/src/mvt/ios/cmd_check_fs.py index 78325ba..e76146e 100644 --- a/src/mvt/ios/cmd_check_fs.py +++ b/src/mvt/ios/cmd_check_fs.py @@ -8,6 +8,7 @@ from typing import Optional from mvt.common.command import Command from mvt.common.indicators import Indicators +from mvt.common.module import MVTModule from .modules.fs import FS_MODULES from .modules.mixed import MIXED_MODULES @@ -29,6 +30,7 @@ class CmdIOSCheckFS(Command): sub_command: bool = False, disable_version_check: bool = False, disable_indicator_check: bool = False, + custom_modules: Optional[list[type[MVTModule]]] = None, ) -> None: super().__init__( target_path=target_path, @@ -42,8 +44,10 @@ class CmdIOSCheckFS(Command): log=log, disable_version_check=disable_version_check, disable_indicator_check=disable_indicator_check, + custom_modules=custom_modules, ) + self.platform = "ios" self.name = "check-fs" self.modules = FS_MODULES + MIXED_MODULES diff --git a/tests/common/test_command.py b/tests/common/test_command.py index 8a19441..865f9f1 100644 --- a/tests/common/test_command.py +++ b/tests/common/test_command.py @@ -42,6 +42,19 @@ class IndependentModule(RecordingModule): pass +class CustomIOSBackupModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class CustomIOSFSModule(RecordingModule): + supported_commands = (("ios", "check-fs"),) + + +class CustomDependsOnBuiltin(RecordingModule): + supported_commands = (("ios", "check-backup"),) + dependencies = (FirstModule,) + + class RecordingCommand(Command): def init(self): self.initialized = True @@ -132,3 +145,46 @@ class TestCommand: assert RecordingModule.run_order == [] assert not hasattr(cmd, "initialized") assert "depends on unavailable module UnavailableModule" in caplog.text + + def test_custom_modules_are_filtered_before_ordering(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [CustomIOSBackupModule, CustomIOSFSModule] + + assert [module.__name__ for module in cmd._ordered_modules()] == [ + "FirstModule", + "CustomIOSBackupModule", + ] + + def test_selected_custom_module_runs(self): + cmd = RecordingCommand(module_name="CustomIOSBackupModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [CustomIOSBackupModule] + + cmd.run() + + assert RecordingModule.run_order == ["CustomIOSBackupModule"] + + def test_selected_unsupported_custom_module_does_not_run(self): + cmd = RecordingCommand(module_name="CustomIOSFSModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [CustomIOSFSModule] + + cmd.run() + + assert RecordingModule.run_order == [] + + def test_custom_module_dependencies_use_topological_order(self): + cmd = RecordingCommand(module_name="CustomDependsOnBuiltin") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [CustomDependsOnBuiltin] + + cmd.run() + + assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"] diff --git a/tests/common/test_module_loader.py b/tests/common/test_module_loader.py new file mode 100644 index 0000000..ce13427 --- /dev/null +++ b/tests/common/test_module_loader.py @@ -0,0 +1,145 @@ +import pytest + +from mvt.common.module import MVTModule +from mvt.common.module_loader import ( + CustomModuleLoadError, + load_custom_modules, + load_custom_modules_from_path, + module_supports_command, +) + + +MODULE_TEMPLATE = """ +from mvt.common.module import MVTModule + + +class {name}(MVTModule): + supported_commands = {supported_commands!r} + + def run(self): + pass + + def check_indicators(self): + pass + + def serialize(self, result): + return None +""" + + +def _write_module(path, name, supported_commands=()): + path.write_text( + MODULE_TEMPLATE.format( + name=name, + supported_commands=supported_commands, + ), + encoding="utf-8", + ) + return path + + +def test_load_custom_modules_from_python_file(tmp_path): + module_path = _write_module(tmp_path / "custom.py", "FileModule") + + modules = load_custom_modules_from_path(str(module_path)) + + assert [module.__name__ for module in modules] == ["FileModule"] + assert issubclass(modules[0], MVTModule) + + +def test_load_custom_modules_from_folder_in_sorted_order(tmp_path): + _write_module(tmp_path / "b_module.py", "BModule") + _write_module(tmp_path / "a_module.py", "AModule") + _write_module(tmp_path / ".hidden.py", "HiddenModule") + _write_module(tmp_path / "__init__.py", "InitModule") + nested = tmp_path / "nested" + nested.mkdir() + _write_module(nested / "nested_module.py", "NestedModule") + + modules = load_custom_modules_from_path(str(tmp_path)) + + assert [module.__name__ for module in modules] == ["AModule", "BModule"] + + +def test_discovery_ignores_imported_base_and_unrelated_classes(tmp_path): + module_path = tmp_path / "custom.py" + module_path.write_text( + """ +from mvt.common.module import MVTModule + + +class Unrelated: + pass + + +class DiscoveredModule(MVTModule): + def run(self): + pass + + def check_indicators(self): + pass + + def serialize(self, result): + return None +""", + encoding="utf-8", + ) + + modules = load_custom_modules_from_path(str(module_path)) + + assert [module.__name__ for module in modules] == ["DiscoveredModule"] + + +def test_load_custom_modules_deduplicates_same_class(tmp_path): + module_path = _write_module(tmp_path / "custom.py", "DuplicateModule") + + modules = load_custom_modules([str(module_path), str(module_path)]) + + assert [module.__name__ for module in modules] == ["DuplicateModule"] + + +def test_load_custom_modules_raises_for_missing_path(tmp_path): + with pytest.raises(CustomModuleLoadError, match="does not exist"): + load_custom_modules_from_path(str(tmp_path / "missing.py")) + + +def test_load_custom_modules_raises_for_import_error(tmp_path): + module_path = tmp_path / "broken.py" + module_path.write_text("raise RuntimeError('broken import')", encoding="utf-8") + + with pytest.raises(CustomModuleLoadError, match="broken import"): + load_custom_modules_from_path(str(module_path)) + + +def test_load_custom_modules_loads_env_folder_first(tmp_path, monkeypatch): + env_folder = tmp_path / "env" + env_folder.mkdir() + cli_folder = tmp_path / "cli" + cli_folder.mkdir() + _write_module(env_folder / "env_module.py", "EnvModule") + _write_module(cli_folder / "cli_module.py", "CliModule") + monkeypatch.setenv("MVT_CUSTOM_MODULES", str(env_folder)) + + modules = load_custom_modules([str(cli_folder)]) + + assert [module.__name__ for module in modules] == ["EnvModule", "CliModule"] + + +def test_module_supports_command_defaults_to_all_commands(tmp_path): + module_path = _write_module(tmp_path / "custom.py", "DefaultModule") + module = load_custom_modules_from_path(str(module_path))[0] + + assert module_supports_command(module, "ios", "check-backup") + assert module_supports_command(module, "android", "check-bugreport") + + +def test_module_supports_command_honors_supported_commands(tmp_path): + module_path = _write_module( + tmp_path / "custom.py", + "SpecificModule", + (("ios", "check-backup"),), + ) + module = load_custom_modules_from_path(str(module_path))[0] + + assert module_supports_command(module, "ios", "check-backup") + assert not module_supports_command(module, "ios", "check-fs") diff --git a/tests/test_custom_modules.py b/tests/test_custom_modules.py new file mode 100644 index 0000000..dcbbbc9 --- /dev/null +++ b/tests/test_custom_modules.py @@ -0,0 +1,193 @@ +from click.testing import CliRunner + +from mvt.android.cli import check_bugreport +from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF +from mvt.android.cmd_check_backup import CmdAndroidCheckBackup +from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport +from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs +from mvt.common.module import MVTModule +from mvt.ios.cli import check_backup, check_fs + + +CUSTOM_MODULE = """ +from mvt.common.module import MVTModule + + +class {name}(MVTModule): + supported_commands = {supported_commands!r} + slug = "{slug}" + + def run(self): + self.results = [{{"message": "custom module ran"}}] + + def check_indicators(self): + pass + + def serialize(self, result): + return None +""" + + +def _write_custom_module(path, name, supported_commands, slug=None): + path.write_text( + CUSTOM_MODULE.format( + name=name, + supported_commands=supported_commands, + slug=slug or name.lower(), + ), + encoding="utf-8", + ) + return path + + +def test_load_module_appears_only_for_supported_cli_command(tmp_path): + module_path = _write_custom_module( + tmp_path / "custom.py", + "IOSBackupOnlyModule", + (("ios", "check-backup"),), + ) + + backup_result = CliRunner().invoke( + check_backup, + ["--list-modules", "--load-module", str(module_path), str(tmp_path)], + ) + fs_result = CliRunner().invoke( + check_fs, + ["--list-modules", "--load-module", str(module_path), str(tmp_path)], + ) + + assert backup_result.exit_code == 0 + assert "IOSBackupOnlyModule" in backup_result.output + assert fs_result.exit_code == 0 + assert "IOSBackupOnlyModule" not in fs_result.output + + +def test_module_option_runs_supported_custom_module(tmp_path): + module_path = _write_custom_module( + tmp_path / "custom.py", + "CustomRunModule", + (("ios", "check-backup"),), + slug="custom_run_module", + ) + output_path = tmp_path / "out" + + result = CliRunner().invoke( + check_backup, + [ + "--module", + "CustomRunModule", + "--load-module", + str(module_path), + "--output", + str(output_path), + str(tmp_path), + ], + ) + + assert result.exit_code == 0 + assert (output_path / "custom_run_module.json").exists() + + +def test_custom_modules_load_from_environment_without_cli_flag(tmp_path, monkeypatch): + custom_modules_path = tmp_path / "custom_modules" + custom_modules_path.mkdir() + _write_custom_module( + custom_modules_path / "env_module.py", + "EnvBugreportModule", + (("android", "check-bugreport"),), + ) + monkeypatch.setenv("MVT_CUSTOM_MODULES", str(custom_modules_path)) + + result = CliRunner().invoke(check_bugreport, ["--list-modules", str(tmp_path)]) + + assert result.exit_code == 0 + assert "EnvBugreportModule" in result.output + + +class NestedBugreportModule(MVTModule): + supported_commands = (("android", "check-bugreport"),) + + +class NestedBackupModule(MVTModule): + supported_commands = (("android", "check-backup"),) + + +class NestedIntrusionLogsModule(MVTModule): + supported_commands = (("android", "check-intrusion-logs"),) + + +class NestedAndroidQFModule(MVTModule): + supported_commands = (("android", "check-androidqf"),) + + +class DummyZip: + def close(self): + pass + + +def test_androidqf_propagates_custom_modules_to_nested_commands(tmp_path, monkeypatch): + records = {} + custom_modules = [ + NestedBugreportModule, + NestedBackupModule, + NestedIntrusionLogsModule, + NestedAndroidQFModule, + ] + cmd = CmdAndroidCheckAndroidQF( + target_path=str(tmp_path), + custom_modules=custom_modules, + ) + + def record_available(name): + def _record(command): + records[name] = [ + module.__name__ + for module in command._available_modules() + if module.__name__.startswith("Nested") + ] + + return _record + + monkeypatch.setattr(cmd, "load_bugreport", lambda: DummyZip()) + monkeypatch.setattr( + CmdAndroidCheckBugreport, + "from_zip", + lambda self, bugreport: None, + ) + monkeypatch.setattr( + CmdAndroidCheckBugreport, + "run", + record_available("bugreport"), + ) + + monkeypatch.setattr(cmd, "load_backup", lambda: b"") + monkeypatch.setattr(CmdAndroidCheckBackup, "from_ab", lambda self, backup: None) + monkeypatch.setattr( + CmdAndroidCheckBackup, + "run", + record_available("backup"), + ) + + intrusion_logs_path = tmp_path / "intrusion_logs" + intrusion_logs_path.mkdir() + setattr(cmd, "_CmdAndroidCheckAndroidQF__format", "dir") + setattr( + cmd, + "_CmdAndroidCheckAndroidQF__files", + ["androidqf/intrusion_logs/security.txt"], + ) + monkeypatch.setattr(cmd, "_read_device_timezone", lambda: None) + monkeypatch.setattr( + CmdAndroidCheckIntrusionLogs, + "run", + record_available("intrusion_logs"), + ) + + assert cmd.run_bugreport_cmd() + assert cmd.run_backup_cmd() + assert cmd.run_intrusion_logs_cmd() + assert records == { + "bugreport": ["NestedBugreportModule"], + "backup": ["NestedBackupModule"], + "intrusion_logs": ["NestedIntrusionLogsModule"], + } From a18e632ec851532c89e25cf7a43947c305ca756d Mon Sep 17 00:00:00 2001 From: besendorf Date: Wed, 1 Jul 2026 12:35:21 +0200 Subject: [PATCH 3/4] Add shell completion command (#817) --- README.md | 17 +++++++ docs/command_completion.md | 49 ++++++++++++++----- src/mvt/android/cli.py | 50 +++++++++++++++++-- src/mvt/common/completion.py | 94 ++++++++++++++++++++++++++++++++++++ src/mvt/common/help.py | 1 + src/mvt/ios/cli.py | 50 +++++++++++++++++-- tests/test_completion.py | 78 ++++++++++++++++++++++++++++++ 7 files changed, 318 insertions(+), 21 deletions(-) create mode 100644 src/mvt/common/completion.py create mode 100644 tests/test_completion.py diff --git a/README.md b/README.md index cb651a1..adcee7a 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,23 @@ For alternative installation options and known issues, please refer to the [docu MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +### Shell completion + +MVT can generate shell completion scripts for Bash, Zsh, and Fish: + +```bash +mvt-ios completion +mvt-android completion +``` + +The commands print setup instructions by default. To generate a completion script directly, pass the shell name: + +```bash +mvt-ios completion bash +mvt-android completion zsh +``` + +MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details. Module-running `check-*` commands can load custom Python modules with `--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the [development documentation](https://docs.mvt.re/en/latest/development/) for diff --git a/docs/command_completion.md b/docs/command_completion.md index 1cd4eb7..7204df5 100644 --- a/docs/command_completion.md +++ b/docs/command_completion.md @@ -1,43 +1,66 @@ -# Command Completion +# Command Completion -MVT utilizes the [Click](https://click.palletsprojects.com/en/stable/) library for creating its command line interface. +MVT utilizes the [Click](https://click.palletsprojects.com/en/stable/) library for creating its command line interface. Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fish. -To enable it, you need to manually register a special function with your shell, which varies depending on the shell you are using. +To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using. -The following describes how to generate the command completion scripts and add them to your shell configuration. +The following describes how to generate the command completion scripts and add them to your shell configuration. > **Note: You will need to start a new shell for the changes to take effect.** ### For Bash ```bash -# Generates bash completion scripts -echo "$(_MVT_IOS_COMPLETE=bash_source mvt-ios)" > ~/.mvt-ios-complete.bash && -echo "$(_MVT_ANDROID_COMPLETE=bash_source mvt-android)" > ~/.mvt-android-complete.bash +# Generate bash completion scripts +mvt-ios completion bash > ~/.mvt-ios-complete.bash +mvt-android completion bash > ~/.mvt-android-complete.bash ``` Add the following to `~/.bashrc`: ```bash # source mvt completion scripts -. ~/.mvt-ios-complete.bash && . ~/.mvt-android-complete.bash +[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash +[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash ``` ### For Zsh ```bash -# Generates zsh completion scripts -echo "$(_MVT_IOS_COMPLETE=zsh_source mvt-ios)" > ~/.mvt-ios-complete.zsh && -echo "$(_MVT_ANDROID_COMPLETE=zsh_source mvt-android)" > ~/.mvt-android-complete.zsh +# Generate zsh completion scripts +mvt-ios completion zsh > ~/.mvt-ios-complete.zsh +mvt-android completion zsh > ~/.mvt-android-complete.zsh ``` Add the following to `~/.zshrc`: ```bash # source mvt completion scripts -. ~/.mvt-ios-complete.zsh && . ~/.mvt-android-complete.zsh +[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh +[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh ``` +### For Fish + +```bash +# Generate fish completion scripts +mkdir -p ~/.config/fish/completions +mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish +mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish +``` + +Fish loads completion files from `~/.config/fish/completions` automatically. + +### Automatic Installation + +MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`: + +```bash +mvt-ios completion bash --install +mvt-android completion bash --install +``` + +Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`. + For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion). - diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index 364f4ef..d13f86c 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -8,6 +8,12 @@ import logging import click from mvt.common.cmd_check_iocs import CmdCheckIOCS +from mvt.common.completion import ( + SUPPORTED_SHELLS, + completion_instructions, + generate_completion_script, + install_completion_script, +) from mvt.common.help import ( HELP_MSG_ANDROID_BACKUP_PASSWORD, HELP_MSG_CHECK_ADB_REMOVED, @@ -17,6 +23,7 @@ from mvt.common.help import ( HELP_MSG_CHECK_BUGREPORT, HELP_MSG_CHECK_IOCS, HELP_MSG_CHECK_INTRUSION_LOGS, + HELP_MSG_COMPLETION, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_HASHES, @@ -85,10 +92,11 @@ def cli(ctx, disable_update_check, disable_indicator_update_check): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + if ctx.invoked_subcommand != "completion": + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== @@ -99,6 +107,40 @@ def version(): return +# ============================================================================== +# Command: completion +# ============================================================================== +@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) +@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) +@click.option( + "--install", + is_flag=True, + help="Write completion files and update shell configuration.", +) +@click.pass_context +def completion(ctx, shell, install): + program_name = "mvt-android" + + if shell is None: + if install: + raise click.UsageError("A shell is required when using --install.") + click.echo(completion_instructions(program_name)) + return + + root_cli = ctx.find_root().command + + if install: + script_path = install_completion_script(root_cli, program_name, shell) + click.echo(f"Installed {shell} completion to {script_path}") + if shell in ("bash", "zsh"): + click.echo(f"Updated ~/.{shell}rc") + else: + click.echo("Fish loads completion files automatically.") + return + + click.echo(generate_completion_script(root_cli, program_name, shell)) + + # ============================================================================== # Command: check-adb (removed) # ============================================================================== diff --git a/src/mvt/common/completion.py b/src/mvt/common/completion.py new file mode 100644 index 0000000..6466a6d --- /dev/null +++ b/src/mvt/common/completion.py @@ -0,0 +1,94 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from pathlib import Path +import shlex + +import click +from click.shell_completion import get_completion_class + + +SUPPORTED_SHELLS = ("bash", "zsh", "fish") + + +def completion_instructions(program_name: str) -> str: + return f"""Shell completion for {program_name} + +Print a completion script: + {program_name} completion bash > ~/.{program_name}-complete.bash + {program_name} completion zsh > ~/.{program_name}-complete.zsh + mkdir -p ~/.config/fish/completions + {program_name} completion fish > ~/.config/fish/completions/{program_name}.fish + +Load the generated Bash script from ~/.bashrc: + [ -f ~/.{program_name}-complete.bash ] && . ~/.{program_name}-complete.bash + +Load the generated Zsh script from ~/.zshrc: + [ -f ~/.{program_name}-complete.zsh ] && . ~/.{program_name}-complete.zsh + +Fish loads completion files from ~/.config/fish/completions automatically. + +To write these files and update Bash/Zsh shell configuration automatically: + {program_name} completion bash --install + {program_name} completion zsh --install + {program_name} completion fish --install +""" + + +def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str: + completion_class = get_completion_class(shell) + if completion_class is None: + raise click.ClickException(f"Unsupported shell: {shell}") + + complete_var = f"_{program_name.upper().replace('-', '_')}_COMPLETE" + return completion_class(cli, {}, program_name, complete_var).source() + + +def install_completion_script( + cli: click.Command, + program_name: str, + shell: str, +) -> Path: + script = generate_completion_script(cli, program_name, shell) + script_path = _completion_script_path(program_name, shell) + script_path.parent.mkdir(parents=True, exist_ok=True) + script_path.write_text(script, encoding="utf-8") + + if shell in ("bash", "zsh"): + _install_shell_source_line(program_name, shell, script_path) + + return script_path + + +def _completion_script_path(program_name: str, shell: str) -> Path: + home = Path.home() + + if shell == "fish": + return home / ".config" / "fish" / "completions" / f"{program_name}.fish" + + return home / f".{program_name}-complete.{shell}" + + +def _install_shell_source_line(program_name: str, shell: str, script_path: Path) -> None: + shell_config_path = Path.home() / f".{shell}rc" + source_line = ( + f"[ -f {shlex.quote(str(script_path))} ] && " + f". {shlex.quote(str(script_path))}" + ) + block = ( + f"# MVT shell completion for {program_name}\n" + f"{source_line}\n" + ) + + if shell_config_path.exists(): + shell_config = shell_config_path.read_text(encoding="utf-8") + if source_line in shell_config: + return + else: + shell_config = "" + + separator = "" if not shell_config or shell_config.endswith("\n") else "\n" + with shell_config_path.open("a", encoding="utf-8") as handle: + handle.write(f"{separator}{block}") diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 4cba131..5a2dc25 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -21,6 +21,7 @@ HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators" HELP_MSG_STIX2 = "Download public STIX2 indicators" HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check" HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check" +HELP_MSG_COMPLETION = "Generate or install shell completion" # IOS Specific HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup" diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index d9f634c..176b036 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -11,6 +11,12 @@ import click from rich.prompt import Prompt from mvt.common.cmd_check_iocs import CmdCheckIOCS +from mvt.common.completion import ( + SUPPORTED_SHELLS, + completion_instructions, + generate_completion_script, + install_completion_script, +) from mvt.common.logo import logo from mvt.common.options import MutuallyExclusiveOption from mvt.common.updates import IndicatorsUpdates @@ -40,6 +46,7 @@ from mvt.common.help import ( HELP_MSG_CHECK_IOS_BACKUP, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, + HELP_MSG_COMPLETION, ) from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from .cmd_check_backup import CmdIOSCheckBackup @@ -91,10 +98,11 @@ def cli(ctx, disable_update_check, disable_indicator_update_check): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + if ctx.invoked_subcommand != "completion": + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== @@ -105,6 +113,40 @@ def version(): return +# ============================================================================== +# Command: completion +# ============================================================================== +@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) +@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) +@click.option( + "--install", + is_flag=True, + help="Write completion files and update shell configuration.", +) +@click.pass_context +def completion(ctx, shell, install): + program_name = "mvt-ios" + + if shell is None: + if install: + raise click.UsageError("A shell is required when using --install.") + click.echo(completion_instructions(program_name)) + return + + root_cli = ctx.find_root().command + + if install: + script_path = install_completion_script(root_cli, program_name, shell) + click.echo(f"Installed {shell} completion to {script_path}") + if shell in ("bash", "zsh"): + click.echo(f"Updated ~/.{shell}rc") + else: + click.echo("Fish loads completion files automatically.") + return + + click.echo(generate_completion_script(root_cli, program_name, shell)) + + # ============================================================================== # Command: decrypt-backup # ============================================================================== diff --git a/tests/test_completion.py b/tests/test_completion.py new file mode 100644 index 0000000..48c0177 --- /dev/null +++ b/tests/test_completion.py @@ -0,0 +1,78 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.ios.cli import cli as ios_cli + + +class TestCompletionCommand: + def test_completion_prints_instructions_by_default(self): + runner = CliRunner() + result = runner.invoke(ios_cli, ["completion"]) + + assert result.exit_code == 0 + assert "Shell completion for mvt-ios" in result.output + assert "mvt-ios completion bash > ~/.mvt-ios-complete.bash" in result.output + assert "Mobile Verification Toolkit" not in result.output + + def test_completion_prints_bash_script(self): + runner = CliRunner() + result = runner.invoke(ios_cli, ["completion", "bash"]) + + assert result.exit_code == 0 + assert "_MVT_IOS_COMPLETE=bash_complete" in result.output + assert "complete -o nosort" in result.output + assert "mvt-ios" in result.output + assert "Mobile Verification Toolkit" not in result.output + + def test_completion_prints_fish_script(self): + runner = CliRunner() + result = runner.invoke(android_cli, ["completion", "fish"]) + + assert result.exit_code == 0 + assert "_MVT_ANDROID_COMPLETE=fish_complete" in result.output + assert "complete --no-files --command mvt-android" in result.output + assert "Mobile Verification Toolkit" not in result.output + + def test_completion_install_updates_bashrc_once(self, tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + runner = CliRunner() + + result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + assert result.exit_code == 0 + + script_path = tmp_path / ".mvt-ios-complete.bash" + bashrc_path = tmp_path / ".bashrc" + assert script_path.exists() + assert "_MVT_IOS_COMPLETE=bash_complete" in script_path.read_text( + encoding="utf-8" + ) + bashrc = bashrc_path.read_text(encoding="utf-8") + assert "[ -f" in bashrc + assert ".mvt-ios-complete.bash" in bashrc + + result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + assert result.exit_code == 0 + assert bashrc_path.read_text(encoding="utf-8") == bashrc + + def test_completion_install_fish_does_not_update_shell_rc( + self, tmp_path, monkeypatch + ): + monkeypatch.setenv("HOME", str(tmp_path)) + runner = CliRunner() + + result = runner.invoke(android_cli, ["completion", "fish", "--install"]) + + assert result.exit_code == 0 + script_path = ( + tmp_path / ".config" / "fish" / "completions" / "mvt-android.fish" + ) + assert script_path.exists() + assert "_MVT_ANDROID_COMPLETE=fish_complete" in script_path.read_text( + encoding="utf-8" + ) + assert not (tmp_path / ".fishrc").exists() From f5b0a3cd914ef3e924d3c185e6e92d3878b28380 Mon Sep 17 00:00:00 2001 From: "Nimrod B." Date: Wed, 1 Jul 2026 12:36:56 +0200 Subject: [PATCH 4/4] WIP: Addition of a timer to virustotal checks (#593) * Add delay option to virustotal checks (#408) * Fix missing delay argument * Fix VirusTotal delay handling * Fix mypy type for VirusTotal package map --------- Co-authored-by: Janik Besendorf --- docs/android/methodology.md | 16 +++++ src/mvt/android/cli.py | 10 +++ .../android/modules/androidqf/aqf_packages.py | 65 ++++++++++++++++++- src/mvt/common/help.py | 2 + src/mvt/common/virustotal.py | 53 +++++++++++++++ tests/android_androidqf/test_packages.py | 55 ++++++++++++++++ 6 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 src/mvt/common/virustotal.py diff --git a/docs/android/methodology.md b/docs/android/methodology.md index fc1a6c7..797ada8 100644 --- a/docs/android/methodology.md +++ b/docs/android/methodology.md @@ -38,6 +38,22 @@ By separating artifact collection from forensic analysis, this approach ensures For more information, refer to the [AndroidQF project documentation](https://github.com/mvt-project/androidqf). +### VirusTotal package lookups + +AndroidQF records APK file hashes in `packages.json`. MVT can optionally look up non-system APK hashes on VirusTotal while checking an AndroidQF acquisition: + +```bash +MVT_VT_API_KEY= mvt-android check-androidqf --virustotal /path/to/androidqf-output +``` + +The `--virustotal` option is disabled by default because it sends APK hashes to VirusTotal and requires network access. It uses the `VT_API_KEY` MVT configuration value, which can also be provided through the `MVT_VT_API_KEY` environment variable. + +To avoid exhausting free VirusTotal API quotas, MVT waits 16 seconds between package hash requests by default. Use `--delay` to change the delay, or `--delay 0` to disable throttling: + +```bash +mvt-android check-androidqf --virustotal --delay 30 /path/to/androidqf-output +``` + ## Android Intrusion Logs On devices where the user has opted into Android's [**Advanced Protection Mode**](https://support.google.com/android/answer/16339980) and turned on the optional Intrusion Logging featrue, Android can create and archive structured *Intrusion Logs* in an encrypted format. These logs record DNS queries, outbound network connections, process starts, ADB activity and other security-relevant events, and are a high-fidelity complement to the rest of an AndroidQF acquisition. The logs are generated on-device and encrypted before being stored in the Google account associated with the device. The encryption key is protected by the user device PIN. The intrusion log data is not accessible to Google. diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index d13f86c..1123742 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -23,6 +23,7 @@ from mvt.common.help import ( HELP_MSG_CHECK_BUGREPORT, HELP_MSG_CHECK_IOCS, HELP_MSG_CHECK_INTRUSION_LOGS, + HELP_MSG_DELAY_CHECKS, HELP_MSG_COMPLETION, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, HELP_MSG_DISABLE_UPDATE_CHECK, @@ -36,6 +37,7 @@ from mvt.common.help import ( HELP_MSG_STIX2, HELP_MSG_VERBOSE, HELP_MSG_VERSION, + HELP_MSG_VIRUS_TOTAL, ) from mvt.common.logo import logo from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules @@ -310,6 +312,10 @@ def check_backup( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) +@click.option("--virustotal", "-V", is_flag=True, help=HELP_MSG_VIRUS_TOTAL) +@click.option( + "--delay", "-d", type=click.IntRange(min=0), default=16, help=HELP_MSG_DELAY_CHECKS +) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @@ -323,6 +329,8 @@ def check_androidqf( module, load_module, hashes, + virustotal, + delay, non_interactive, backup_password, verbose, @@ -340,6 +348,8 @@ def check_androidqf( module_options={ "interactive": not non_interactive, "backup_password": cli_load_android_backup_password(log, backup_password), + "virustotal": virustotal, + "virustotal_delay": delay, }, disable_version_check=_get_disable_flags(ctx)[0], disable_indicator_check=_get_disable_flags(ctx)[1], diff --git a/src/mvt/android/modules/androidqf/aqf_packages.py b/src/mvt/android/modules/androidqf/aqf_packages.py index 264fd1e..294ad53 100644 --- a/src/mvt/android/modules/androidqf/aqf_packages.py +++ b/src/mvt/android/modules/androidqf/aqf_packages.py @@ -5,8 +5,11 @@ import json import logging +import time from typing import Optional +from rich.progress import track + from mvt.android.utils import ( BROWSER_INSTALLERS, PLAY_STORE_INSTALLERS, @@ -15,7 +18,8 @@ from mvt.android.utils import ( SYSTEM_UPDATE_PACKAGES, THIRD_PARTY_STORE_INSTALLERS, ) -from mvt.common.module_types import ModuleResults +from mvt.common.module_types import ModuleAtomicResult, ModuleResults +from mvt.common.virustotal import VTNoKey, VTQuotaExceeded, virustotal_lookup from .base import AndroidQFModule @@ -125,6 +129,65 @@ class AQFPackages(AndroidQFModule): ) break + if self.module_options.get("virustotal", False): + self.check_virustotal( + delay=self.module_options.get("virustotal_delay", 0) + ) + + def check_virustotal(self, delay: int = 0) -> None: + files_by_hash: dict[ + str, list[tuple[ModuleAtomicResult, ModuleAtomicResult]] + ] = {} + for package in self.results: + if package.get("system", False): + continue + + for package_file in package.get("files", []): + file_hash = package_file.get("sha256") + if not file_hash: + continue + + files_by_hash.setdefault(file_hash, []).append((package, package_file)) + + total_hashes = len(files_by_hash) + if total_hashes == 0: + return + + progress_desc = f"Looking up {total_hashes} package files on VirusTotal..." + for index, file_hash in enumerate( + track(files_by_hash, description=progress_desc) + ): + try: + results = virustotal_lookup(file_hash) + except VTNoKey as exc: + self.log.warning("%s", exc) + return + except VTQuotaExceeded as exc: + self.log.warning("Unable to continue VirusTotal lookups: %s", exc) + break + + if index < total_hashes - 1 and delay > 0: + time.sleep(delay) + + if not results: + continue + + attributes = results.get("attributes", {}) + stats = attributes.get("last_analysis_stats", {}) + positives = stats.get("malicious", 0) + total = len(attributes.get("last_analysis_results", {})) + detection = f"{positives}/{total}" + + for package, package_file in files_by_hash[file_hash]: + package_file["virustotal"] = detection + if positives > 0: + self.alertstore.high( + f'VirusTotal flagged package "{package["name"]}" file ' + f'"{package_file["path"]}" with {detection} detections', + "", + package, + ) + def run(self) -> None: packages = self._get_files_by_pattern("*/packages.json") if not packages: diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 5a2dc25..90e71ca 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -53,3 +53,5 @@ HELP_MSG_CHECK_BUGREPORT = "Check an Android Bug Report" HELP_MSG_CHECK_ANDROID_BACKUP = "Check an Android Backup" HELP_MSG_CHECK_ANDROIDQF = "Check data collected with AndroidQF" HELP_MSG_CHECK_INTRUSION_LOGS = "Check Android Intrusion Logging files" +HELP_MSG_VIRUS_TOTAL = "Check package APK hashes on VirusTotal" +HELP_MSG_DELAY_CHECKS = "Delay in seconds between VirusTotal requests" diff --git a/src/mvt/common/virustotal.py b/src/mvt/common/virustotal.py new file mode 100644 index 0000000..9a5df32 --- /dev/null +++ b/src/mvt/common/virustotal.py @@ -0,0 +1,53 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +from typing import Any, Optional + +import requests + +from .config import settings + +log = logging.getLogger(__name__) + + +class VTNoKey(Exception): + pass + + +class VTQuotaExceeded(Exception): + pass + + +def virustotal_lookup(file_hash: str) -> Optional[dict[str, Any]]: + if not settings.VT_API_KEY: + raise VTNoKey( + "No VirusTotal API key provided: to use VirusTotal lookups please set " + "MVT_VT_API_KEY or VT_API_KEY in the MVT configuration file" + ) + + headers = { + "User-Agent": "VirusTotal", + "Content-Type": "application/json", + "x-apikey": settings.VT_API_KEY, + } + res = requests.get( + f"https://www.virustotal.com/api/v3/files/{file_hash}", + headers=headers, + timeout=settings.NETWORK_TIMEOUT, + ) + + if res.status_code == 200: + report = res.json() + return report["data"] + + if res.status_code == 404: + log.info("Could not find results for file with hash %s", file_hash) + elif res.status_code == 429: + raise VTQuotaExceeded("You have exceeded the quota for your VirusTotal API key") + else: + raise RuntimeError(f"Unexpected response from VirusTotal: {res.status_code}") + + return None diff --git a/tests/android_androidqf/test_packages.py b/tests/android_androidqf/test_packages.py index ae0ae62..a879e88 100644 --- a/tests/android_androidqf/test_packages.py +++ b/tests/android_androidqf/test_packages.py @@ -7,8 +7,11 @@ import logging from pathlib import Path import pytest +from click.testing import CliRunner +from mvt.android.cli import check_androidqf from mvt.android.modules.androidqf.aqf_packages import AQFPackages +from mvt.android.modules.androidqf import aqf_packages as aqf_packages_module from mvt.common.module import run_module from ..utils import get_android_androidqf, list_files @@ -132,3 +135,55 @@ class TestAndroidqfPackages: possible_detected_app[0].matched_indicator.value == "c7e56178748be1441370416d4c10e34817ea0c961eb636c8e9d98e0fd79bf730" ) + + def test_virustotal_delays_after_missing_result(self, monkeypatch): + lookups = [] + sleeps = [] + + def fake_virustotal_lookup(file_hash): + lookups.append(file_hash) + if file_hash == "missing_hash": + return None + return { + "attributes": { + "last_analysis_stats": {"malicious": 1}, + "last_analysis_results": {"engine": {}}, + } + } + + monkeypatch.setattr( + aqf_packages_module, "virustotal_lookup", fake_virustotal_lookup + ) + monkeypatch.setattr(aqf_packages_module.time, "sleep", sleeps.append) + + module = AQFPackages( + module_options={"virustotal": True, "virustotal_delay": 16}, + results=[ + { + "name": "org.example", + "installer": "com.android.vending", + "disabled": False, + "system": False, + "files": [ + {"path": "/data/app/missing.apk", "sha256": "missing_hash"}, + {"path": "/data/app/found.apk", "sha256": "found_hash"}, + ], + } + ], + ) + + module.check_indicators() + + assert lookups == ["missing_hash", "found_hash"] + assert sleeps == [16] + assert module.results[0]["files"][1]["virustotal"] == "1/1" + assert len(module.alertstore.alerts) == 1 + + +def test_check_androidqf_rejects_negative_virustotal_delay(data_path): + runner = CliRunner() + + result = runner.invoke(check_androidqf, ["--delay", "-1", data_path]) + + assert result.exit_code == 2 + assert "Invalid value for '--delay'" in result.output