diff --git a/.github/workflows/publish-release-docker.yml b/.github/workflows/publish-release-docker.yml index eddca22..b3fde3a 100644 --- a/.github/workflows/publish-release-docker.yml +++ b/.github/workflows/publish-release-docker.yml @@ -59,6 +59,9 @@ jobs: type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }} type=sha,suffix=${{ matrix.platform.tag-suffix }} type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }} + # Register emulators so Buildx can build the ARM64 images on the AMD64 runner. + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 # This step sets up some additional capabilities to generate the provenance and sbom attestations - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -71,6 +74,7 @@ jobs: with: file: ${{ matrix.platform.dockerfile }} context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index da06064..58bc216 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,12 +119,23 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y \ adb \ + binutils \ default-jre-headless \ + file \ + jq \ + less \ libcurl4 \ + libimage-exiftool-perl \ libssl3 \ libusb-1.0-0 \ + moreutils \ + p7zip-full \ python3 \ - sqlite3 + ripgrep \ + sqlite3 \ + tree \ + unzip \ + xxd COPY --from=build-libplist /build / COPY --from=build-libimobiledevice-glue /build / COPY --from=build-libtatsu /build / @@ -144,7 +155,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2 RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc # Generate adb key folder -RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc +RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc RUN mkdir /root/.android # Setup investigations environment diff --git a/Dockerfile.android b/Dockerfile.android index 2c060db..2b74611 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -31,7 +31,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2 RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc # Generate adb key folder -RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc +RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc RUN mkdir /root/.android ENTRYPOINT [ "/usr/local/bin/mvt-android" ] diff --git a/README.md b/README.md index 65fcdee..98e90e3 100644 --- a/README.md +++ b/README.md @@ -58,34 +58,33 @@ For alternative installation options and known issues, please refer to the [docu ## Usage -MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion`, `plugins` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) + +Pass `--verbose` to any of the three commands, before the command name (`mvt-ios --verbose check-backup ...`), for debug output. The `--verbose` option the `check-*` commands accept after their name still works but is kept for compatibility only and will be removed in a future release. ### Shell completion -MVT can generate shell completion scripts for Bash, Zsh, and Fish: +MVT can generate a shell completion script for Bash, Zsh, and Fish which covers `mvt`, `mvt-ios` and `mvt-android`: ```bash -mvt-ios completion -mvt-android completion +mvt completion ``` -The commands print setup instructions by default. To generate a completion script directly, pass the shell name: +The command prints setup instructions by default. To generate the completion script directly, pass the shell name: ```bash -mvt-ios completion bash -mvt-android completion zsh +mvt completion bash ``` MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details. -Module-running `check-*` commands can load custom Python modules with -`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the -[development documentation](https://docs.mvt.re/en/latest/development/) for -details. -Users can also add top-level commands to `mvt-ios` and `mvt-android` from -installed Python packages or local files and folders. See the -[custom CLI command documentation](https://docs.mvt.re/en/latest/custom_commands/) -for the plugin entry points and `--load-command` interface. +Plugin packages extend MVT with additional forensic modules, which run inside +the `check-*` commands, and with top-level commands on `mvt`, `mvt-ios` and +`mvt-android`. See the +[development documentation](https://docs.mvt.re/en/latest/development/) for +writing and installing them, and the +[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/) +for the entry points a package registers commands in. ## License diff --git a/docs/android/intrusion_logs.md b/docs/android/intrusion_logs.md index da8fc83..16c6bf1 100644 --- a/docs/android/intrusion_logs.md +++ b/docs/android/intrusion_logs.md @@ -58,7 +58,7 @@ mvt-android check-intrusion-logs --output /path/to/results/ /path/to/intrusion-l | `-l, --list-modules` | List the available intrusion-log modules and exit. | | `-m, --module NAME` | Run a single module (e.g. `DnsEvent`) instead of all of them. | | `-t, --timezone TZ` | IANA timezone name for the device (e.g. `Europe/Paris`). When set, event timestamps are converted to the device's local time instead of UTC. | -| `-v, --verbose` | Verbose logging. | +| `-v, --verbose` | Verbose logging. Kept for compatibility and to be removed in a future release: pass `--verbose` to `mvt-android` itself instead. | ## Modules diff --git a/docs/command_completion.md b/docs/command_completion.md index 7204df5..a51cc7a 100644 --- a/docs/command_completion.md +++ b/docs/command_completion.md @@ -6,61 +6,61 @@ Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fi To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using. -The following describes how to generate the command completion scripts and add them to your shell configuration. +`mvt completion` generates one script which covers `mvt`, `mvt-ios` and `mvt-android`. The following describes how to generate that script and add it to your shell configuration. > **Note: You will need to start a new shell for the changes to take effect.** ### For Bash ```bash -# Generate bash completion scripts -mvt-ios completion bash > ~/.mvt-ios-complete.bash -mvt-android completion bash > ~/.mvt-android-complete.bash +# Generate the bash completion script +mvt completion bash > ~/.mvt-complete.bash ``` Add the following to `~/.bashrc`: ```bash -# source mvt completion scripts -[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash -[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash +# source the mvt completion script +[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash ``` ### For Zsh ```bash -# Generate zsh completion scripts -mvt-ios completion zsh > ~/.mvt-ios-complete.zsh -mvt-android completion zsh > ~/.mvt-android-complete.zsh +# Generate the zsh completion script +mvt completion zsh > ~/.mvt-complete.zsh ``` Add the following to `~/.zshrc`: ```bash -# source mvt completion scripts -[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh -[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh +# source the mvt completion script +[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh ``` ### For Fish ```bash -# Generate fish completion scripts -mkdir -p ~/.config/fish/completions -mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish -mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish +# Generate the fish completion script +mkdir -p ~/.config/fish/conf.d +mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish ``` -Fish loads completion files from `~/.config/fish/completions` automatically. +Fish loads the files in `~/.config/fish/conf.d` automatically. ### Automatic Installation MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`: ```bash -mvt-ios completion bash --install -mvt-android completion bash --install +mvt completion bash --install ``` -Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`. +Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/conf.d` and changes no shell configuration. + +!!! note + + Earlier versions generated one script per command, with `mvt-ios completion` + and `mvt-android completion`. Files written by them keep working. When you + switch to the single script, remove the old files and the lines which load + them from your shell configuration. For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion). - diff --git a/docs/custom_commands.md b/docs/custom_commands.md deleted file mode 100644 index ab89fd9..0000000 --- a/docs/custom_commands.md +++ /dev/null @@ -1,107 +0,0 @@ -# Custom CLI Commands - -MVT can load additional top-level commands into `mvt-ios` and `mvt-android`. -Custom commands are different from [custom forensic modules](development.md#custom-modules): -commands add new CLI operations, while modules add analysis steps to existing -`check-*` commands. - -!!! warning - - Custom commands run as trusted Python code inside the MVT process. Install - or load commands only from sources you trust. MVT does not sandbox - third-party commands, and the MVT maintainers do not maintain them. - -## Install a Command Package - -Python packages can register a Click command or group for either MVT CLI. A -minimal package can expose this command from `my_mvt_plugin.py`: - -```python -import click - - -@click.command() -@click.argument("path", type=click.Path(exists=True)) -def summarize(path): - """Summarize an acquisition.""" - click.echo(f"Summarizing {path}") -``` - -Register the object in the package's `pyproject.toml`. The entry-point name is -the command users invoke: - -```toml -[project.entry-points."mvt.ios.cli_plugins"] -summarize = "my_mvt_plugin:summarize" - -[project.entry-points."mvt.android.cli_plugins"] -summarize = "my_mvt_plugin:summarize" -``` - -Use only the iOS or Android group if the command is platform-specific. After -installing the package in the same environment as MVT, it appears directly in -the appropriate CLI: - -```bash -mvt-ios summarize ./ios-backup -mvt-android summarize ./androidqf-output -``` - -For a `pipx` installation of MVT, inject the plugin into MVT's environment: - -```bash -pipx inject mvt my-mvt-plugin -``` - -When MVT is installed in an active virtual environment, install the plugin with -`pip` in that environment. - -## Load a Command File - -For local commands that are not packaged, create a Python file that exports one -Click command or group named `cli`: - -```python -import click - - -@click.command("case-summary") -@click.argument("path", type=click.Path(exists=True)) -def cli(path): - """Summarize a case directory.""" - click.echo(f"Summarizing {path}") -``` - -Pass the file before the custom command name: - -```bash -mvt-ios --load-command ./case_summary.py case-summary ./ios-backup -``` - -`--load-command` can be repeated and also accepts a folder. MVT loads -non-hidden top-level `*.py` files in sorted order and skips `__init__.py`. -Every loaded file must export one `cli` object. - -To load a file or folder on every invocation, set the platform-specific -environment variable: - -```bash -export MVT_IOS_CUSTOM_COMMANDS=./ios_commands -export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands -``` - -## Naming and Errors - -Built-in MVT commands cannot be replaced. External command names must also be -unique; when installed packages or environment paths collide, MVT keeps the -first command and logs a warning. A collision from an explicit -`--load-command` is a usage error. - -A package entry point or environment command that cannot be imported appears -as a marked broken command without preventing other MVT commands from working. -Invoke that command to see its package or file source and the underlying error. -An invalid command supplied explicitly with `--load-command` fails immediately -with a usage error. - -Installed command packages use the entry-point name as the CLI command name. -The entry point must resolve to a `click.Command` or `click.Group`. diff --git a/docs/development.md b/docs/development.md deleted file mode 100644 index ffe1265..0000000 --- a/docs/development.md +++ /dev/null @@ -1,131 +0,0 @@ -# Development - -The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software. - -## Local environment - -MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run: - -```bash -make install -``` - -## Testing - -MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can -be run automatically with: - -```bash -make check -``` - -Run these tests before making new commits or opening pull requests. - -## Module dependencies - -Modules can require other modules to run first by declaring their classes in -`dependencies`. The command runner uses a stable topological ordering, so the -existing module list order is preserved wherever dependency constraints allow. - -```python -class DependentModule(MVTModule): - dependencies = (PrerequisiteModule,) - - def run(self): - prerequisite_results = self.get_dependency_results(PrerequisiteModule) -``` - -Selecting a single module also runs its transitive dependencies. If a dependency -is unavailable or the dependency graph contains a cycle, the command logs a -warning and does not run any modules. - -## Custom modules - -Module-running `check-*` commands can load custom modules from Python files that -are not installed as part of MVT. Load one file with: - -```bash -mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup -``` - -You can also load a folder. MVT loads non-hidden top-level `*.py` files in -sorted order and skips `__init__.py`: - -```bash -mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump -``` - -Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This -folder is loaded before any `--load-module` path: - -```bash -MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip -``` - -Custom modules are normal `MVTModule` subclasses: - -```python -from mvt.common.module import MVTModule - - -class ExampleCustomModule(MVTModule): - supported_commands = (("ios", "check-backup"), ("ios", "check-fs")) - slug = "example_custom_module" - - def run(self): - self.results = [{"message": "custom module ran"}] - - def check_indicators(self): - pass - - def serialize(self, result): - return None -``` - -Use `supported_commands` to declare the platform/command pairs a module -supports. Empty `supported_commands` means the module will not run and MVT logs -a warning. This explicit declaration is required for every command. Supported -pairs are: - -```python -("ios", "check-backup") -("ios", "check-fs") -("ios", "check-iocs") -("ios", "check-sysdiagnose") -("android", "check-backup") -("android", "check-bugreport") -("android", "check-androidqf") -("android", "check-intrusion-logs") -("android", "check-iocs") -``` - -Custom modules can depend on existing MVT module classes. Dependencies are -resolved with the same ordering logic as built-in modules, and custom modules -are appended after built-ins before ordering: - -```python -from mvt.common.module import MVTModule -from mvt.ios.modules.backup.manifest import Manifest - - -class DependentCustomModule(MVTModule): - supported_commands = (("ios", "check-backup"),) - dependencies = (Manifest,) - - def run(self): - manifest_results = self.get_dependency_results(Manifest) - self.results = [{"manifest_entries": len(manifest_results)}] -``` - -## Profiling - -Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be -take to avoid inefficient code paths as we add new modules. - -MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable. - -```bash -MVT_PROFILE=1 dev/mvt-ios check-backup test_backup -``` - -Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT. diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md new file mode 100644 index 0000000..7aac490 --- /dev/null +++ b/docs/development/custom_commands.md @@ -0,0 +1,202 @@ +# Custom CLI Commands + +MVT can load additional top-level commands into `mvt`, `mvt-ios` and +`mvt-android`. A command package chooses which of the three each of its +commands is added to. +Custom commands are different from [custom forensic modules](index.md#custom-modules): +commands add new CLI operations, while modules add analysis steps to existing +`check-*` commands. + +!!! warning + + Custom commands run as trusted Python code inside the MVT process. Install + or load commands only from sources you trust. MVT does not sandbox + third-party commands, and the MVT maintainers do not maintain them. + +## Install a Command Package + +Python packages can register a Click command or group on one or more of the MVT CLIs. +A minimal package can expose this command from `my_mvt_plugin.py`: + +```python +import click + + +@click.command() +@click.argument("path", type=click.Path(exists=True)) +def summarize(path): + """Summarize an acquisition.""" + click.echo(f"Summarizing {path}") +``` + +Log through `get_plugin_logger(__name__)` from `mvt.plugin`. Records logged +through `logging.getLogger(__name__)` do not reach `command.log`, and MVT's +console handler does not show them. + +Register the object in the package's `pyproject.toml`. The entry-point name is +the command users invoke: + +```toml +[project.entry-points."mvt.ios.cli_plugins"] +summarize = "my_mvt_plugin:summarize" + +[project.entry-points."mvt.android.cli_plugins"] +summarize = "my_mvt_plugin:summarize" +``` + +Each entry-point group adds the command to one CLI: `mvt.ios.cli_plugins` to +`mvt-ios`, `mvt.android.cli_plugins` to `mvt-android` and `mvt.cli_plugins` to +`mvt`. Register the command in the group of every CLI which should offer it: a +platform-specific command belongs in one platform group, and a command which +handles acquisitions of both platforms, as above, in both. After installing the +package in the same environment as MVT, the command appears directly in those +CLIs: + +```bash +mvt-ios summarize ./ios-backup +mvt-android summarize ./androidqf-output +``` + +For a `pipx` installation of MVT, inject the plugin into MVT's environment: + +```bash +pipx inject mvt my-mvt-plugin +``` + +When MVT is installed in an active virtual environment, install the plugin with +`pip` in that environment. `mvt plugins list` shows the installed packages and +the commands they add, see [Managing Plugins](plugins.md). + +Command packages that need their own settings, such as an API key, should store +them in a namespaced [plugin configuration file](plugin_configuration.md) +rather than in MVT's own `config.yaml`. + +### Commands on `mvt` + +A MVT plugin command can also add sub-commands to the base `mvt` command. This can be used for commands which are not tied to a particular forensic platform: + +```toml +[project.entry-points."mvt.cli_plugins"] +my-plugin = "my_mvt_plugin:my_plugin" +``` + +Commands in this group are added to `mvt` only, so this one is invoked as +`mvt my-plugin`. A command on `mvt` has nothing but its name to say which +plugin it belongs to, so name it after the plugin, and make it a Click group +when the plugin has several operations to offer, such as +`mvt my-plugin configure`. + +## Developing a Command Locally + +A package is how a command is distributed. While a command is being written, +MVT can load it straight from its file instead, so the package need not be +reinstalled after every change; an editable install of the package does the +same through its entry points. Create a Python file that exports one Click +command or group named `cli`: + +```python +import click + + +@click.command("case-summary") +@click.argument("path", type=click.Path(exists=True)) +def cli(path): + """Summarize a case directory.""" + click.echo(f"Summarizing {path}") +``` + +Pass the file before the custom command name: + +```bash +mvt-ios --load-command ./case_summary.py case-summary ./ios-backup +``` + +`--load-command` can be repeated and also accepts a folder. MVT loads +non-hidden top-level `*.py` files in sorted order and skips `__init__.py`. +Every loaded file must export one `cli` object. + +To load a file or folder on every invocation, set the environment variable of +the CLI the commands belong on. Like the entry-point groups, each variable adds +its commands to one CLI only: + +```bash +export MVT_CUSTOM_COMMANDS=./commands +export MVT_IOS_CUSTOM_COMMANDS=./ios_commands +export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands +``` + +## Building a Module-Running Command + +A command which runs forensic modules over an acquisition subclasses `Command`. +`Command` creates the output folder and writes `command.log`. It orders the +modules, resolves their dependencies and runs them. It writes the result files, +`alerts.json` and `info.json`. The subclass sets `platform`, `name` and +`modules`: + +```python +from mvt.plugin import Command, MVTModule, convert_unix_to_iso, get_plugin_logger + +log = get_plugin_logger(__name__) + + +class APKManifest(MVTModule): + supported_commands = (("android", "check-apks"),) + + def run(self): + self.results = [{"checked_at": convert_unix_to_iso(0)}] + + +class CmdCheckAPKs(Command): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.platform = "android" + self.name = "check-apks" + self.modules = [APKManifest] +``` + +`platform` and `name` are the pair the modules declare in +`supported_commands`. `modules` lists the module classes the command runs, +imported directly. A command which runs modules of another plugin depends on +that package in `pyproject.toml` and imports them the same way. `init()`, +`module_init(module)` and `finish()` are optional hooks. `run()` calls them +before the run, before each module and after the run. + +Wrap the command in a Click command with an `--output` option. Run it, then +print the alert summary: + +```python +import click + + +@click.command("check-apks") +@click.option("--output", "-o", type=click.Path(exists=False)) +@click.argument("TARGET_PATH", type=click.Path(exists=True)) +def cli(output, target_path): + cmd = CmdCheckAPKs(target_path=target_path, results_path=output) + log.info("Checking APK files at path: %s", target_path) + cmd.run() + cmd.show_alerts_brief() +``` + +The `--verbose` option of `mvt`, `mvt-ios` and `mvt-android` applies to the +command. The command defines no `--verbose` option of its own. The pair a +plugin command adds is not listed anywhere in MVT. Name it in the plugin's +README. + +## Naming and Errors + +Built-in MVT commands cannot be replaced. External command names must also be +unique on each CLI; when installed packages or environment paths collide, MVT +keeps the first command and logs a warning. The environment path of a CLI is +registered before its installed packages, so a command loaded from there wins a +collision with a package. A collision from an explicit `--load-command` is a +usage error. + +A package entry point or environment command that cannot be imported appears +as a marked broken command without preventing other MVT commands from working. +Invoke that command to see its package or file source and the underlying error. +An invalid command supplied explicitly with `--load-command` fails immediately +with a usage error. + +Installed command packages use the entry-point name as the CLI command name. +The entry point must resolve to a `click.Command` or `click.Group`. diff --git a/docs/development/index.md b/docs/development/index.md new file mode 100644 index 0000000..8b80238 --- /dev/null +++ b/docs/development/index.md @@ -0,0 +1,391 @@ +# Development + +The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software. + +## Local environment + +MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run: + +```bash +make install +``` + +## Testing + +MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can +be run automatically with: + +```bash +make check +``` + +Run these tests before making new commits or opening pull requests. + +## Module dependencies + +Modules can require other modules to run first by declaring their classes in +`dependencies`. The command runner uses a stable topological ordering, so the +existing module list order is preserved wherever dependency constraints allow. + +```python +class DependentModule(MVTModule): + dependencies = (PrerequisiteModule,) + + def run(self): + prerequisite_results = self.get_dependency_results(PrerequisiteModule) +``` + +Selecting a single module also runs its transitive dependencies. + +A module can only depend on modules the command it runs in also has. When a +declared dependency is not among them, the command logs a warning naming the +module and the missing dependency, skips that module and everything depending +on it, and runs the rest of the analysis. Selecting such a module with +`--module` therefore leaves nothing to run, which the warning explains. + +A cycle in the dependency graph is a programming error rather than a +configuration problem: the command logs a warning and runs no modules at all. + +## Custom modules + +MVT's module-running `check-*` commands can run forensic modules which are not +part of MVT. Custom modules are distributed as plugin packages. A Python +package installed next to MVT registers its modules through an entry point. +The modules then load automatically in every command they support, see +[Installed module packages](#installed-module-packages). `mvt plugins list` +shows the installed packages and where each one was installed from. + +MVT can also load module files by path, with `--load-module` and +`MVT_CUSTOM_MODULES`, see +[Developing modules locally](#developing-modules-locally). This can be used +while writing a module. Use a Python package to distribute one. + +A custom module declares in `supported_commands` the platform and command +pairs it runs in. A module with empty `supported_commands` does not run and +MVT logs a warning. The nine pairs are: + +```python +("ios", "check-backup") +("ios", "check-fs") +("ios", "check-iocs") +("ios", "check-sysdiagnose") +("android", "check-backup") +("android", "check-bugreport") +("android", "check-androidqf") +("android", "check-intrusion-logs") +("android", "check-iocs") +``` + +`check-iocs` re-checks stored results rather than an acquisition. It matches +every `.json` file in the results folder to the module with that slug. +It then runs that module's `check_indicators()` again. A module which +implements `check_indicators()` is included in `check-iocs` for its platform. +It does not need to declare the `check-iocs` pair. + +### Writing a module + +A module subclasses `MVTModule` or one of the base classes below and +implements `run()`. `check_indicators()` and `serialize()` are optional. The +first matches results against IOCs or detections. The second returns timeline +records. + +```python +from mvt.plugin import IOSExtraction, convert_unix_to_iso + + +class ExampleCustomModule(IOSExtraction): + supported_commands = ( + ("ios", "check-backup"), + ("ios", "check-fs"), + ) + slug = "example_custom_module" + + def run(self): + self.results = [{"checked_at": convert_unix_to_iso(0)}] + + def check_indicators(self): + pass + + def serialize(self, result): + return None +``` + +The base classes are: + +- `MVTModule`: the base of every module. It provides `self.results`, + `self.alertstore`, `self.log`, `self.indicators` and + `get_dependency_results()`. Subclass it directly for a module which reads + only the results of other modules. +- `IOSExtraction`: `("ios", "check-backup")` and `("ios", "check-fs")`. Adds + `_find_ios_database()`, which locates a module's database in a backup or in + a filesystem dump and repairs it if it is malformed. Adds + `_get_backup_files_from_manifest()`, `_get_backup_file_from_id()` and + `_get_fs_files_from_patterns()`. Adds `_open_sqlite_db()`, which opens a + database read-only. +- `SysdiagnoseExtraction`: `("ios", "check-sysdiagnose")`. MVT extracts the + archive and calls `from_sysdiagnose_folder()` before `run()`. The module + reads files with `_get_files_by_pattern()` and `_get_file_content()`. + `ips_files` lists the crash reports. See + [Check an iOS Sysdiagnose](../ios/sysdiagnose.md). +- `AndroidQFModule`: `("android", "check-androidqf")`. MVT calls `from_dir()` + or `from_zip()` with the file list of the acquisition. The module reads + files with `_get_files_by_pattern()` and `_get_file_content()`. + `_get_device_timezone()` returns the device timezone. +- `AndroidBackupModule`: `("android", "check-backup")`. MVT calls `from_dir()` + or `from_ab()`. The module reads files with `_get_files_by_pattern()` and + `_get_file_content()`. +- `BugReportModule`: `("android", "check-bugreport")`. MVT calls `from_dir()` + or `from_zip()`. The module reads files with `_get_files_by_pattern()`, + `_get_files_by_patterns()` and `_get_file_content()`. + `_get_dumpstate_file()` returns the dumpstate file, and + `_get_file_modification_time()` the modification time of a file. + +The underscore-named helpers are internal to the base classes. Plugin modules +can call them. Their names and signatures can change between releases. Read +the base class in `src/mvt/ios/modules` or `src/mvt/android/modules` before +relying on one. + +### Depending on a built-in module + +A module which post-processes records generated by one or more built-in MVT +modules must declare the source modules in `dependencies`. It reads their +results with `get_dependency_results()`. Import the class from its family +package: `mvt.ios.modules.backup`, `mvt.ios.modules.fs`, +`mvt.ios.modules.mixed`, `mvt.android.modules.androidqf`, +`mvt.android.modules.backup`, `mvt.android.modules.bugreport` or +`mvt.android.modules.intrusion_logs`. + +```python +from mvt.ios.modules.backup import Manifest +from mvt.plugin import MVTModule + + +class DependentCustomModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + dependencies = (Manifest,) + + def run(self): + manifest_results = self.get_dependency_results(Manifest) + self.results = [{"manifest_entries": len(manifest_results)}] +``` + +Dependencies are ordered as for the built-in modules, with custom modules +appended after the built-ins, see [Module dependencies](#module-dependencies). +A dependency has to run in every command the module supports. Where it does +not, MVT skips the module with a warning. + +`get_dependency_results()` returns the plain dictionaries the module produced. +They are the same records it writes to `.json`. Typed results per module +are planned. + +### Replacing a built-in module + +A custom module which extends a built-in one runs alongside it, and when the +two share a slug they write to the same results file. MVT warns about that +collision, naming both modules, where each came from and the file the later one +overwrites, but it runs them both. Set `replaces` to the class the module +supersedes to take that module's place instead: when both are available to a +command, the named module is dropped from the run. + +```python +from mvt.ios.modules.backup import Manifest as BuiltinManifest + + +class Manifest(BuiltinManifest): + supported_commands = (("ios", "check-backup"),) + replaces = BuiltinManifest +``` + +Keeping the class name of the replaced module, as above, keeps the name +`--module` selects the module by and the slug its results file is named after. +A replacement with a different class name writes to a file named after its own +slug, unless it sets `slug` to the slug of the module it replaces, and +`--module` still selects it by the name of the replaced module, which MVT logs. +Taking over the slug of a replaced module is not a collision and is not warned +about, because that module is no longer part of the run. + +Modules which depend on the replaced module receive the replacement instead, so +`get_dependency_results(BuiltinManifest)` returns the replacement's results. A +module must not depend on the module it replaces: that module does not run, so +the replacement has to produce the data itself, and MVT logs a warning when a +module declares both. + +A replacement takes on the obligations of the module it replaces. If it +declares a dependency the command does not provide, it is skipped like any +other module with an unavailable dependency, and the module it replaced stays +out of the run: neither of them produces results, and the modules depending on +the replaced module are skipped as well. + +Subclassing the replaced module is not required, but a replacement which is not +a subclass is logged with a warning, because its results may not be what the +modules depending on the replaced module expect. Naming a module the command +does not run has no effect, and modules which replace each other in a cycle all +keep running and replace nothing. Every applied substitution is logged, so it +is recorded in `command.log` when the command runs with an `--output` folder. + +Every command resolves replacements on its own. `check-iocs` matches stored +results files against the slugs of the modules available for that command. A +replacement which subclasses the module it replaces inherits its +`check_indicators()`. It is then part of `check-iocs` for its platform and +re-checks the results file named after its slug. A replacement with no +`check_indicators()` is not part of `check-iocs`. The built-in module it +replaced re-checks the file. `check-iocs` matches `--module` on the class name +only. Pass a differently named replacement's own name there, not the name of +the module it replaces. + +### Importing from MVT + +Import from `mvt.plugin` if it has what you need. The names it exports are +kept working on a best-effort basis. A change to one of them is announced in +the release notes. Anything else in `mvt` can be imported too, but may change +between releases without notice. The plugin interface is best effort. + +`mvt.plugin` exports the base classes above and `Command`, `Alert` and +`AlertLevel`, the result types, `DatabaseNotFoundError` and +`DatabaseCorruptedError`, the timestamp converters, the settings API of +[Plugin Configuration](plugin_configuration.md), MVT's own `settings`, +`get_plugin_logger()` and `MVT_VERSION`. `src/mvt/plugin.py` holds the list. +Read MVT's `settings` for values such as `NETWORK_ACCESS_ALLOWED` and +`NETWORK_TIMEOUT`. Plugin values go in the plugin's own settings file. + +## Installed module packages + +Python packages can register modules so they load automatically in every +module-running `check-*` command, without `--load-module` or +`MVT_CUSTOM_MODULES`. Register an entry point in the `mvt.modules` group in +the package's `pyproject.toml`: + +```toml +[project.entry-points."mvt.modules"] +mvt-plugin-example-org = "mvt_plugin_example_org:get_modules" +``` + +The entry point must resolve to an iterable of `MVTModule` subclasses, or to +a callable returning one: + +```python +from mvt.plugin import MVTModule + + +class PackagedModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + + def run(self): + self.results = [{"message": "packaged module ran"}] + + +def get_modules() -> list[type[MVTModule]]: + return [PackagedModule] +``` + +`get_modules()` is the package's module list, written by hand. A package which +keeps its modules in separate files imports each class there and lists it. A +module missing from the list does not load. + +Installed modules follow the same rules as other custom modules: each module +must declare `supported_commands`, and dependencies are resolved with the +standard ordering logic. A broken entry point is skipped with a warning and +does not prevent MVT from running. As with custom commands, installed module +packages run as trusted code inside the MVT process, so install only packages +from sources you trust. + +For a `pipx` installation of MVT, inject the package into MVT's environment: + +```bash +pipx inject mvt mvt-plugin-example-org +``` + +Module packages that need their own settings, such as an API key, should store +them in a namespaced [plugin configuration file](plugin_configuration.md) +rather than in MVT's own `config.yaml`. + +### Naming module packages + +Name module packages `mvt-plugin-` (import package `mvt_plugin_`), +and include the name of the publishing organization or author so packages from +different groups do not collide: for example, an organisation's custom modules +would be distributed as `mvt-plugin-example-org` with the import package +`mvt_plugin_example_org`. + +The prefix makes module packages easy to find on PyPI and keeps their import +names from clashing with unrelated Python packages. It is a convention, not a +technical requirement: modules load through the `mvt.modules` entry point +regardless of what the package is called, and existing packages with other +names keep working. Note that the prefix is also not a mark of authenticity — +anyone can publish a package with any available name, so vet a module package +and its publisher before installing it, whatever it is called. + +### Module logging + +Modules log through `self.log`, and MVT names the logger for where the module +came from. MVT's own modules log under their dotted path (for example +`mvt.ios.modules.mixed.whatsapp`). Everything external is namespaced under +`mvt.ext` to keep it visually distinct from built-in modules and isolated from +MVT's internal logger tree: + +- Installed packages log under `mvt.ext.`, with the `mvt_plugin_` + prefix stripped: modules in `mvt_plugin_example_org` log as + `mvt.ext.example_org.*`. +- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as + `mvt.ext.`. + +Outside a module class, for example in a custom command line handler, log +through `get_plugin_logger(__name__)`. It returns a logger in the same +namespace. + +## Developing modules locally + +While a module is being written, load it from its file. `--load-module` takes a +Python file, or a folder of them, on every module-running command, and can be +repeated: + +```bash +mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup +``` + +For a folder, MVT loads its non-hidden top-level `*.py` files in sorted order +and skips `__init__.py`. `MVT_CUSTOM_MODULES` names a folder to load on every +module-running command, before any `--load-module` path: + +```bash +MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip +``` + +Files loaded this way follow the same rules as packaged modules. +`--list-modules` reports them with the SHA-256 hash of the file in place of a +version. An editable install of the package (`pip install -e .`) also works: +the modules load through the entry point, and `mvt plugins list` shows the +package with the `local` origin. + +Loading by path is for development. Move a module into a package once it +works. + +## Auditing loaded modules + +Because installed module packages load automatically, MVT records where every +module came from: + +- `--list-modules` groups the available modules by source: MVT itself + (with its version), each installed package (with its version and, when + installed directly from a repository, the commit), and each file loaded + with `--load-module` or `MVT_CUSTOM_MODULES` (with the SHA-256 hash of the + file). +- When a command runs with an `--output` folder, the `command.log` file + records one line per module source with the source's version or hash and + the list of modules loaded from it. +- `mvt plugins list` lists the installed packages, where each of them + was installed from and how many modules it contributes, see + [Managing Plugins](plugins.md). + +## Profiling + +Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be +take to avoid inefficient code paths as we add new modules. + +MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable. + +```bash +MVT_PROFILE=1 dev/mvt-ios check-backup test_backup +``` + +Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT. diff --git a/docs/development/plugin_configuration.md b/docs/development/plugin_configuration.md new file mode 100644 index 0000000..62d35a7 --- /dev/null +++ b/docs/development/plugin_configuration.md @@ -0,0 +1,157 @@ +# Plugin Configuration + +Plugin packages that add [custom CLI commands](custom_commands.md) or +[modules](index.md#installed-module-packages) often need to store their +own settings, such as an API key, a server URL or the timestamp of the last +synchronization. MVT provides a namespaced settings base class so each plugin +keeps its configuration in its own file, and a data folder for anything else a +plugin needs to keep on disk. + +!!! warning + + Do not write plugin settings to MVT's own `config.yaml`. MVT rewrites that + file with the settings it knows about every time it starts, so any other + section is deleted. + +## Where Settings Are Stored + +Each plugin gets one YAML file in a `plugins` folder next to MVT's own +configuration: + +``` +~/.config/mvt/plugins/.yaml +``` + +The exact parent folder follows the platform convention used for MVT's +`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use +`plugin_config_path()` from `mvt.plugin` instead of building the path by hand. + +Plugin names must be lowercase and may only contain letters, digits and dashes, +matching the `mvt-plugin-` package naming convention. MVT creates the +`plugins` folder with `0700` permissions and writes the settings files with +`0600` permissions, because they commonly hold credentials. Files are written +through a temporary file and moved into place, so an interrupted save never +leaves a partially written settings file behind. + +## Plugin Data Folder + +Everything else a plugin keeps on disk, such as a cache, a downloaded artifact +or synchronization state, belongs in the folder returned by the `data_folder()` +class method of the plugin's settings class, or by `plugin_data_folder()` from +`mvt.plugin`, called with the plugin name if the plugin has no settings class: + +``` +~/.local/share/mvt/plugin-data// # Linux +~/Library/Application Support/mvt/plugin-data// # macOS +``` + +The folder sits beside MVT's own data, such as the downloaded indicators. It is +created if it is missing, with `0700` permissions. Asking for it again returns +the same path and leaves the contents alone, so a plugin can ask for it every +time it needs the folder. `ExamplePluginSettings` below is the settings class +defined in the next section: + +```python +import os + + +def cache_path() -> str: + folder = ExamplePluginSettings.data_folder() + return os.path.join(folder, "virustotal_lookups_cache.json") +``` + +A plugin which has no settings class calls +`plugin_data_folder("example-plugin")` instead. + +Do not fall back on a path of your own such as `~/.cache/example-plugin`: it +is a Linux-only convention, and MVT will not create it for you. + +## Defining Plugin Settings + +Subclass `MVTPluginSettings`, set `plugin_name` and declare typed fields with +defaults: + +```python +from typing import Optional + +from mvt.plugin import MVTPluginSettings + + +class ExamplePluginSettings(MVTPluginSettings): + plugin_name = "example-plugin" + + API_KEY: Optional[str] = None + MAX_RESULTS: int = 25 + LAST_SYNC: Optional[str] = None +``` + +`load()` returns the current settings and `save()` writes them back: + +```python +from datetime import datetime, timezone + +import click + +from mvt.plugin import plugin_env_prefix + + +def sync(): + settings = ExamplePluginSettings.load() + if not settings.API_KEY: + prefix = plugin_env_prefix(settings.plugin_name) + raise click.ClickException( + f"No API key configured. Set {prefix}API_KEY or " + "run 'example-plugin configure'." + ) + + settings.LAST_SYNC = datetime.now(timezone.utc).isoformat() + settings.save() +``` + +The message builds the variable name with `plugin_env_prefix()`, see +[Environment Variables](#environment-variables). + +A missing settings file is not an error: the plugin then runs on the field +defaults and on whatever the environment provides. `save()` only persists the +values that differ from the defaults, and it never touches MVT's `config.yaml`. +A settings file that cannot be parsed, or that does not hold a mapping of +setting names to values, raises a `PluginConfigLoadError` naming the file. + +Every subclass that sets its own `plugin_name` gets its own file and its own +environment namespace. A subclass that does not redefine `plugin_name` inherits +it, and therefore shares the file and the environment variables of its parent +class. + +## Environment Variables + +Every field can also be set with an environment variable. The prefix is +`MVT_PLUGIN_`, followed by the plugin name upper-cased with dashes replaced by +underscores, followed by the field name. For the example above: + +```bash +export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=... +export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50 +``` + +Do not repeat the plugin name in a field name: a plugin named `example-scanner` +with a `SCANNER_API_KEY` field asks the user for +`MVT_PLUGIN_EXAMPLE_SCANNER_SCANNER_API_KEY`. Name the field `API_KEY`. + +Settings resolve in this order, from highest to lowest priority: + +1. Arguments passed to the settings class directly, such as + `ExamplePluginSettings(API_KEY="...")` +2. Environment variables +3. The plugin's YAML file +4. The field defaults declared on the settings class + +!!! tip + + On shared or multi-user machines, prefer passing API keys through + environment variables rather than saving them to the plugin file. `save()` + skips every value that the environment currently supplies, so a credential + provided that way is not copied into the settings file when a plugin saves + an unrelated setting. + +Unknown keys in a plugin's YAML file are ignored, so a settings file written by +a newer version of a plugin does not break an older one. diff --git a/docs/development/plugins.md b/docs/development/plugins.md new file mode 100644 index 0000000..0085975 --- /dev/null +++ b/docs/development/plugins.md @@ -0,0 +1,116 @@ +# Managing Plugins + +Plugin packages extend MVT with additional +[forensic modules](index.md#custom-modules) and +[CLI commands](custom_commands.md). Because installed packages load +automatically, `mvt plugins` audits what is installed and checks whether +updates are available. The command lives on `mvt` only, although the packages +it lists extend `mvt-ios` and `mvt-android` too. + +## List Installed Plugins + +```bash +mvt plugins list +``` + +``` + Installed MVT plugins +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓ +┃ Name ┃ Version ┃ Origin ┃ Modules ┃ Commands ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩ +│ mvt-plugin-example │ 1.2.0 │ pypi │ 4 │ summarize │ +│ mvt-plugin-research │ 0.1.0 │ git+3f9a1c7d │ 2 │ - │ +│ mvt-plugin-local │ 0.0.1 │ local │ 1 │ triage │ +└─────────────────────┴─────────┴──────────────┴─────────┴───────────┘ +``` + +The origin records where each package was installed from: `pypi` for a package +installed from a package index, `git+` for a package installed directly +from a repository, and `local` for a package installed from a local folder or +archive rather than from an index, including an editable development install. +The last two columns show how many forensic modules the package contributes and +which CLI commands it adds. + +A plugin whose modules cannot be imported is listed with `error` in the +`Modules` column rather than breaking the listing. + +## Check for Updates + +```bash +mvt plugins check-updates +``` + +``` +Plugin updates available: + mvt-plugin-example 1.2.0 → 1.3.0 + Upgrade with: pip install -U mvt-plugin-example + +MVT does not install plugin updates. Run the command above when you decide to +upgrade. +``` + +Packages installed from a package index are compared against the latest release +published for them. A package which was never published, for example a plugin +distributed only within an organization, is skipped silently. + +!!! note + + Packages shown with the `pypi` origin are compared against + [PyPI](https://pypi.org), whichever index they were installed from. A + plugin installed from a private index under a name which also exists on + PyPI is therefore compared against the unrelated public package of that + name. Give plugins published to a private index a name which is not taken + on PyPI, and treat an unexpected update suggestion as a reason to check + where the package would come from. + +!!! warning + + MVT never installs or upgrades a plugin itself, it only prints the command + which does. Upgrading a plugin in the middle of an investigation changes + the modules producing the results, and a plugin runs as trusted code inside + the MVT process, so pulling in a new version is a decision for the analyst + to make deliberately and not a side effect of running a check. + +## Automatic Update Checks + +MVT also reports available plugin updates in the banner printed when a command +starts: + +``` + MVT - Mobile Verification Toolkit + + https://mvt.re + Version: 2026.7.29 + + Plugin updates available: + mvt-plugin-example 1.2.0 → 1.3.0 (pip install -U mvt-plugin-example) +``` + +This check runs at most once every 12 hours. In between checks MVT prints the +findings of the latest check without contacting anything, so a plugin update +stays visible without a lookup on every command. The +`mvt plugins check-updates` command checks immediately, regardless of when the +last check happened. + +The automatic check is skipped when the `--disable-update-check` option is +used, when `NETWORK_ACCESS_ALLOWED` is disabled in the MVT configuration, and +when no plugins are installed. + +## Plugins Installed From a Repository + +A plugin installed with `pip install "mvt-plugin-example @ git+"` is +checked by asking the remote repository which commit the installed revision +points at now. MVT runs git and ssh in batch mode, so a repository which needs +credentials MVT does not already have fails the check instead of prompting for +them. The check is skipped silently when git is not available, when the +repository cannot be reached, and when access to it is denied. + +How the plugin was installed decides what an update means: + +- A plugin installed from a branch is reported as outdated when the branch has + moved past the installed commit. +- A plugin installed from a specific commit or a tag is pinned. It is never + reported as outdated, however far the branch it came from moves on. + +Pinning a plugin to a commit or a tag is therefore the way to keep the modules +used across an investigation stable. diff --git a/docs/index.md b/docs/index.md index fada316..8cd733e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forens It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors. -In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results. +In this documentation you will find instructions on how to install and run the `mvt-ios`, `mvt-android` and `mvt` commands, and guidance on how to interpret the extracted results. ## Resources diff --git a/docs/install.md b/docs/install.md index c08c75d..c024338 100644 --- a/docs/install.md +++ b/docs/install.md @@ -64,7 +64,7 @@ It is recommended to try installing and running MVT from [Windows Subsystem Linu pipx install mvt ``` -You now should have the `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window. +You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window. ### Installing from PyPI directly into a virtual environment You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool: @@ -84,7 +84,7 @@ source env/bin/activate pip install mvt ``` -The `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active. +The `mvt`, `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active. ### Installing from git source with pipx If you want to have the latest features in development, you can install MVT directly from the source code in git. @@ -93,7 +93,7 @@ If you want to have the latest features in development, you can install MVT dire pipx install --force git+https://github.com/mvt-project/mvt.git ``` -You now should have the `mvt-ios` and `mvt-android` utilities installed. +You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. **Notes:** 1. The `--force` flag is necessary to force the reinstallation of the package. diff --git a/docs/iocs.md b/docs/iocs.md index c7586de..5f50eff 100644 --- a/docs/iocs.md +++ b/docs/iocs.md @@ -71,7 +71,7 @@ So far MVT implements only a subset of [STIX2 specifications](https://docs.oasis - [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2). - We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns. -You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT. +You can automatically download the latest public indicator files with the command `mvt download-iocs`. The per-platform forms `mvt-ios download-iocs` and `mvt-android download-iocs` do the same thing. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT. Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs. diff --git a/docs/ios/records.md b/docs/ios/records.md index 016c861..b34183a 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -417,7 +417,21 @@ If indicators are provided through the command-line, they are checked against th Backup: :material-check: Full filesystem dump: :material-check: -This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*. +This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*, along with one record per chat session (marked with `"record_type": "chat_session"`) containing the first and last interaction dates of each conversation. Chat sessions produce `chat_first_message` and `chat_last_message` timeline events, and group chats additionally produce a `group_created` event. A chat session's last-message date can postdate its newest stored message when the most recent messages in the chat were deleted. + +Recent WhatsApp versions key 1:1 chat sessions by an opaque LID identifier rather than the contact's phone number. The module resolves these using the `ZWAPHONENUMBERLIDPAIR` table from the *LID.sqlite* database in the same app group (or from *ChatStorage.sqlite* itself in versions that store it there), populating `partner_resolved_phone_number` on chat session records and using the phone number in timeline events. Each LID-phone number pair is also extracted as a record (`"record_type": "lid_phone_number_pair"`) and produces a `lid_pair_recorded` timeline event marking when WhatsApp learned the association. If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*. +--- + +### `whatsapp_contacts.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extracts WhatsApp contact records from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ContactsV2.sqlite*, including each contact's phone number, WhatsApp and LID identifiers, and the per-contact disappearing messages timer, which is not recorded in *ChatStorage.sqlite*. Each timestamp stored on a contact record produces a timeline event: `disappearing_mode_set` (when the disappearing messages timer was last changed), `about_changed` (when the contact last changed their "about" text), `about_expiration` (when a timed "about" is scheduled to expire) and `contact_last_updated` (when the contact record was last updated). + +This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup. + diff --git a/docs/ios/sysdiagnose.md b/docs/ios/sysdiagnose.md index 6488bc2..d0d84fb 100644 --- a/docs/ios/sysdiagnose.md +++ b/docs/ios/sysdiagnose.md @@ -1,30 +1,32 @@ # Check an iOS Sysdiagnose `mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by -custom MVT modules. MVT does not include built-in sysdiagnose modules. You must -load at least one custom module that explicitly supports this command. +custom MVT modules. MVT does not include built-in sysdiagnose modules. The +command runs the modules of the installed +[plugin packages](../development/index.md#installed-module-packages) which +declare support for it. Install at least one such package first. The command accepts either an extracted sysdiagnose directory or the original gzip-compressed tar archive. ```bash -mvt-ios check-sysdiagnose \ - --load-module ./sysdiagnose_modules.py \ - --output ./results \ +mvt-ios check-sysdiagnose --output ./results \ ./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz ``` Use `--hashes` to include hashes for analyzed files in `info.json`, and -`--list-modules` to display the eligible custom modules without running them. +`--list-modules` to display the eligible modules without running them. ## Writing a custom module -Extend `SysdiagnoseExtraction` to access the archive contents consistently for -both directory and tar inputs. Each module must declare the command explicitly -in `supported_commands`. +Extend `SysdiagnoseExtraction` from `mvt.plugin`, see +[Writing a module](../development/index.md#writing-a-module). The module reads +the archive the same way whether MVT was given a folder or a tar archive. It +declares the command in `supported_commands`. While writing one, +[load it from its file](../development/index.md#developing-modules-locally). ```python -from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction +from mvt.plugin import SysdiagnoseExtraction class ExampleSysdiagnoseModule(SysdiagnoseExtraction): @@ -44,7 +46,9 @@ class ExampleSysdiagnoseModule(SysdiagnoseExtraction): return None ``` -The base class provides `from_sysdiagnose_folder()` and -`from_sysdiagnose_tar()` setup hooks, as well as protected file lookup, file -reading, and timezone extraction helpers. IPS crash-report metadata is exposed -on `ips_files`. +MVT extracts a tar archive first. It calls `from_sysdiagnose_folder()` on each +module before `run()`. `ips_files` lists the IPS crash reports. + +`_get_files_by_pattern()` and `_get_file_content()` are internal helpers of the +base class. Use them to read the archive. Their names and signatures can change +between releases. See `src/mvt/ios/modules/sysdiagnose/base.py`. diff --git a/mkdocs.yml b/mkdocs.yml index 66f3471..66def17 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,7 +31,6 @@ nav: - Introduction: "introduction.md" - Installation: "install.md" - Command Completion: "command_completion.md" - - Custom CLI Commands: "custom_commands.md" - Using Docker: "docker.md" - MVT for iOS: - iOS Forensic Methodology: "ios/methodology.md" @@ -50,5 +49,9 @@ nav: - Check an Android Backup (SMS messages): "android/backup.md" - Check Android Intrusion Logs: "android/intrusion_logs.md" - Indicators of Compromise: "iocs.md" - - Development: "development.md" + - Development: + - Development Instructions: "development/index.md" + - Custom CLI Commands: "development/custom_commands.md" + - Plugin Configuration: "development/plugin_configuration.md" + - Managing Plugins: "development/plugins.md" - License: "license.md" diff --git a/pyproject.toml b/pyproject.toml index 151d4e7..fb90f8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,9 @@ homepage = "https://docs.mvt.re/en/latest/" repository = "https://github.com/mvt-project/mvt" [project.scripts] -mvt-ios = "mvt.ios:cli" -mvt-android = "mvt.android:cli" +mvt = "mvt.cli:main" +mvt-ios = "mvt.ios:main" +mvt-android = "mvt.android:main" [dependency-groups] dev = [ diff --git a/src/mvt/android/__init__.py b/src/mvt/android/__init__.py index 2c05f56..6616bcb 100644 --- a/src/mvt/android/__init__.py +++ b/src/mvt/android/__init__.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from .cli import cli +from .cli import cli, main diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index 22a69e5..b0b0040 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -197,9 +197,14 @@ class TombstoneCrashArtifact(AndroidArtifact): def _load_key_value_line( self, line: str, key: str, destination_key: str, tombstone: dict ) -> bool: - line_key, value = line.split(":", 1) - if line_key != key: - raise ValueError(f"Expected key {key}, got {line_key}") + # The caller matched the key as a bare prefix, so a longer word starting + # with it arrives here: `Caused by: …` inside an abort message reaches + # the `Cause` key. That is a different line, not a broken file — say so + # by declining it, and let the remaining keys have their turn. Raising + # here discarded the whole tombstone, crash and stack trace included. + line_key, separator, value = line.partition(":") + if not separator or line_key != key: + return False value_clean = value.strip().strip("'") if destination_key == "uid": diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index ca7e3dc..7b78a4e 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -14,13 +14,6 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.completion import ( - SUPPORTED_SHELLS, - completion_instructions, - generate_completion_script, - install_completion_script, -) from mvt.common.help import ( HELP_MSG_ANDROID_BACKUP_PASSWORD, HELP_MSG_CHECK_ADB_REMOVED, @@ -31,7 +24,6 @@ from mvt.common.help import ( HELP_MSG_CHECK_IOCS, HELP_MSG_CHECK_INTRUSION_LOGS, HELP_MSG_DELAY_CHECKS, - HELP_MSG_COMPLETION, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_HASHES, @@ -43,23 +35,16 @@ from mvt.common.help import ( HELP_MSG_OUTPUT, HELP_MSG_STIX2, HELP_MSG_VERBOSE, + HELP_MSG_VERBOSE_COMMAND, HELP_MSG_VERSION, HELP_MSG_VIRUS_TOTAL, ) -from mvt.common.logo import logo -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import init_logging, set_verbose_logging -from .cmd_check_androidqf import CmdAndroidCheckAndroidQF -from .cmd_check_backup import CmdAndroidCheckBackup -from .cmd_check_bugreport import CmdAndroidCheckBugreport -from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs -from .modules.intrusion_logs import INTRUSION_LOGS_MODULES -from .modules.androidqf import ANDROIDQF_MODULES -from .modules.backup import BACKUP_MODULES -from .modules.backup.helpers import cli_load_android_backup_password -from .modules.bugreport import BUGREPORT_MODULES +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-android, including by shell completion on +# every keystroke, so importing it must do no more than build the command tree: +# the forensic modules, the backup parsers and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -77,7 +62,14 @@ def _get_disable_flags(ctx): ) +def _get_verbose(ctx): + """Return whether --verbose was passed to the CLI itself.""" + return bool(ctx.obj and ctx.obj.get("verbose", False)) + + def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -97,60 +89,31 @@ def _load_custom_modules(load_module): is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - if ctx.invoked_subcommand != "completion": - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) + + from mvt.common.logo import logo + + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== # Command: version # ============================================================================== -@cli.command("version", help=HELP_MSG_VERSION) +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) def version(): return -# ============================================================================== -# Command: completion -# ============================================================================== -@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) -@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) -@click.option( - "--install", - is_flag=True, - help="Write completion files and update shell configuration.", -) -@click.pass_context -def completion(ctx, shell, install): - program_name = "mvt-android" - - if shell is None: - if install: - raise click.UsageError("A shell is required when using --install.") - click.echo(completion_instructions(program_name)) - return - - root_cli = ctx.find_root().command - - if install: - script_path = install_completion_script(root_cli, program_name, shell) - click.echo(f"Installed {shell} completion to {script_path}") - if shell in ("bash", "zsh"): - click.echo(f"Updated ~/.{shell}rc") - else: - click.echo("Fish loads completion files automatically.") - return - - click.echo(generate_completion_script(root_cli, program_name, shell)) - - # ============================================================================== # Command: check-adb (removed) # ============================================================================== @@ -187,7 +150,7 @@ def check_adb(ctx): default=[], help=HELP_MSG_LOAD_MODULE, ) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BUGREPORT_PATH", type=click.Path(exists=True)) @click.pass_context def check_bugreport( @@ -200,7 +163,9 @@ def check_bugreport( verbose, bugreport_path, ): - set_verbose_logging(verbose) + from .cmd_check_bugreport import CmdAndroidCheckBugreport + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as bug reports are small. cmd = CmdAndroidCheckBugreport( @@ -255,7 +220,7 @@ def check_bugreport( ) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( @@ -269,7 +234,10 @@ def check_backup( verbose, backup_path, ): - set_verbose_logging(verbose) + from .cmd_check_backup import CmdAndroidCheckBackup + from .modules.backup.helpers import cli_load_android_backup_password + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as backups are generally small. @@ -329,7 +297,7 @@ def check_backup( ) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("ANDROIDQF_PATH", type=click.Path(exists=True)) @click.pass_context def check_androidqf( @@ -347,7 +315,10 @@ def check_androidqf( verbose, androidqf_path, ): - set_verbose_logging(verbose) + from .cmd_check_androidqf import CmdAndroidCheckAndroidQF + from .modules.backup.helpers import cli_load_android_backup_password + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdAndroidCheckAndroidQF( @@ -415,7 +386,7 @@ def check_androidqf( "time instead of UTC." ), ) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("LOGS_PATH", type=click.Path(exists=True)) @click.pass_context def check_intrusion_logs( @@ -429,7 +400,9 @@ def check_intrusion_logs( verbose, logs_path, ): - set_verbose_logging(verbose) + from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) module_options = {} @@ -482,6 +455,10 @@ def check_intrusion_logs( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import ANDROID_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -492,9 +469,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): custom_modules=custom_modules, platform="android", ) - cmd.modules = ( - BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES - ) + cmd.modules = ANDROID_CHECK_IOCS_MODULES if list_modules: cmd.list_modules() @@ -510,12 +485,25 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_indicators(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() -register_cli_plugins( - cli, - entry_point_group=ANDROID_CLI_PLUGIN_GROUP, - environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV, -) +# ============================================================================== +# Entry point of the mvt-android console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt-android CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=ANDROID_CLI_PLUGIN_GROUP, + environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/android/cmd_check_androidqf.py b/src/mvt/android/cmd_check_androidqf.py index ffa3e13..99ac5a3 100644 --- a/src/mvt/android/cmd_check_androidqf.py +++ b/src/mvt/android/cmd_check_androidqf.py @@ -292,9 +292,7 @@ class CmdAndroidCheckAndroidQF(Command): try: cmd.from_ab(backup) except InvalidAndroidBackup as exc: - self.log.warning( - "Skipping backup modules as backup.ab is malformed: %s", exc - ) + self.log.warning("Skipping backup modules: %s", exc) return False cmd.run() diff --git a/src/mvt/android/cmd_check_backup.py b/src/mvt/android/cmd_check_backup.py index b75bb34..94be4ee 100644 --- a/src/mvt/android/cmd_check_backup.py +++ b/src/mvt/android/cmd_check_backup.py @@ -87,11 +87,15 @@ class CmdAndroidCheckBackup(Command): if header["encryption"] != "none": password = prompt_or_load_android_backup_password(log, self.module_options) if not password: + if self.sub_command: + raise InvalidAndroidBackup("No backup password provided") log.critical("No backup password provided.") sys.exit(1) try: tardata = parse_backup_file(ab_file_bytes, password=password) except InvalidBackupPassword: + if self.sub_command: + raise InvalidAndroidBackup("Invalid backup password") log.critical("Invalid backup password") sys.exit(1) except AndroidBackupParsingError as exc: diff --git a/src/mvt/android/command_modules.py b/src/mvt/android/command_modules.py new file mode 100644 index 0000000..9b6b56e --- /dev/null +++ b/src/mvt/android/command_modules.py @@ -0,0 +1,23 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Module lists an mvt-android command composes from more than one family. + +Commands whose modules are one family read that family directly. check-iocs +re-checks stored results, so it has to know every module that could have +written one, and both the CLI and any other code needing that answer share +the list from here rather than each concatenating their own. +""" + +from mvt.common.module import MVTModule + +from .modules.androidqf import ANDROIDQF_MODULES +from .modules.backup import BACKUP_MODULES +from .modules.bugreport import BUGREPORT_MODULES +from .modules.intrusion_logs import INTRUSION_LOGS_MODULES + +ANDROID_CHECK_IOCS_MODULES: list[type[MVTModule]] = ( + BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES +) diff --git a/src/mvt/cli.py b/src/mvt/cli.py new file mode 100644 index 0000000..6e8c667 --- /dev/null +++ b/src/mvt/cli.py @@ -0,0 +1,110 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import click + +from mvt.common.cli_plugins import ( + MVT_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, + load_cli_commands_option, + register_cli_plugins, +) +from mvt.common.cmd_plugins import plugins +from mvt.common.completion import completion +from mvt.common.help import ( + HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, + HELP_MSG_DISABLE_UPDATE_CHECK, + HELP_MSG_STIX2, + HELP_MSG_VERBOSE, + HELP_MSG_VERSION, +) +from mvt.common.logo import logo +from mvt.common.updates import IndicatorsUpdates +from mvt.common.utils import init_logging, set_verbose_logging + +init_logging() + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +# ============================================================================== +# Main +# ============================================================================== +@click.group(invoke_without_command=True) +@load_cli_commands_option +@click.option( + "--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK +) +@click.option( + "--disable-indicator-update-check", + is_flag=True, + help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, +) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.pass_context +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): + """Mobile Verification Toolkit. + + The 'mvt-ios' and 'mvt-android' CLI commands are used to perform + forensic analysis on IOS and Android devices. + """ + ctx.ensure_object(dict) + ctx.obj["disable_version_check"] = disable_update_check + ctx.obj["disable_indicator_check"] = disable_indicator_update_check + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) + if ctx.invoked_subcommand != "completion": + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +# ============================================================================== +# Command: download-iocs +# ============================================================================== +@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) +def download_iocs(): + ioc_updates = IndicatorsUpdates() + ioc_updates.update() + + +# ============================================================================== +# Command: completion +# ============================================================================== +cli.add_command(completion) + + +# ============================================================================== +# Command: version +# ============================================================================== +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) +def version(): + return + + +# The plugins command is registered as a built-in command, before any external +# command, so that an installed package can never replace it. +cli.add_command(plugins) + + +# ============================================================================== +# Entry point of the mvt console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/common/cli_plugins.py b/src/mvt/common/cli_plugins.py index acf4c6d..17766ac 100644 --- a/src/mvt/common/cli_plugins.py +++ b/src/mvt/common/cli_plugins.py @@ -15,10 +15,20 @@ from typing import Iterable import click +# This module is imported by every CLI at start-up, and by shell completion on +# every keystroke, so it must stay cheap: nothing here may import the module +# machinery (mvt.common.module_loader and what it pulls in). + IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins" ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins" +# Commands in this group are registered on the platform-neutral mvt command only. +NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins" +MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS" MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS" MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS" +# Prefix of the import name given to a command file loaded from a path. Shared +# with module_loader, which recognises such files when naming their loggers. +CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" log = logging.getLogger(__name__) @@ -54,7 +64,7 @@ class BrokenPluginCommand(click.Command): def _module_name_for_path(path: Path) -> str: digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] - return f"_mvt_custom_command_{path.stem}_{digest}" + return f"{CUSTOM_COMMAND_MODULE_PREFIX}{path.stem}_{digest}" def _iter_command_files(path: Path) -> Iterable[Path]: @@ -255,6 +265,16 @@ def register_cli_plugins( entry_point_group: str, environment_variable: str, ) -> None: + """Register the external commands of one CLI on its group. + + Each CLI has one entry-point group and one environment variable of its + own, so a command package chooses the CLIs its commands are added to. + + :param group: CLI group to register the external commands on. + :param entry_point_group: Entry-point group of the CLI. + :param environment_variable: Name of the environment variable holding a + path to load commands from. + """ environment_path = os.environ.get(environment_variable) if environment_path: register_cli_commands_from_path(group, environment_path) diff --git a/src/mvt/common/cmd_check_iocs.py b/src/mvt/common/cmd_check_iocs.py index c227659..c2dbdcd 100644 --- a/src/mvt/common/cmd_check_iocs.py +++ b/src/mvt/common/cmd_check_iocs.py @@ -9,6 +9,7 @@ from typing import Optional from mvt.common.command import Command from mvt.common.module import MVTModule +from mvt.common.module_loader import get_module_logger from mvt.common.utils import exec_or_profile log = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class CmdCheckIOCS(Command): ) m = iocs_module.from_json( - file_path, log=logging.getLogger(iocs_module.__module__) + file_path, log=get_module_logger(iocs_module) ) if not m: log.warning("No result from this module, skipping it") diff --git a/src/mvt/common/cmd_plugins.py b/src/mvt/common/cmd_plugins.py new file mode 100644 index 0000000..21619eb --- /dev/null +++ b/src/mvt/common/cmd_plugins.py @@ -0,0 +1,205 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import importlib.metadata +import logging +from typing import Optional + +import click +from rich.console import Console +from rich.table import Table + +from .cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from .config import settings +from .help import ( + HELP_MSG_PLUGINS, + HELP_MSG_PLUGINS_CHECK_UPDATES, + HELP_MSG_PLUGINS_LIST, +) +from .module import MVTModule +from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url +from .updates import ( + SHORT_COMMIT_LENGTH, + PluginUpdates, + installed_plugin_distributions, +) + +log = logging.getLogger(__name__) + +CLI_PLUGIN_GROUPS = ( + IOS_CLI_PLUGIN_GROUP, + ANDROID_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +def _entry_points(group: str) -> list[importlib.metadata.EntryPoint]: + try: + return list(importlib.metadata.entry_points(group=group)) + except Exception as exc: + log.warning("Unable to discover the entry points in group %s: %s", group, exc) + return [] + + +def _entry_point_distribution( + entry_point: importlib.metadata.EntryPoint, +) -> Optional[str]: + dist = getattr(entry_point, "dist", None) + if dist is None: + return None + try: + return dist.name + except Exception: + return None + + +def _distribution_version(dist: importlib.metadata.Distribution) -> str: + try: + return dist.version or "unknown" + except Exception: + return "unknown" + + +def _distribution_origin(dist: importlib.metadata.Distribution) -> str: + """Describe where a plugin package was installed from.""" + direct_url = distribution_direct_url(dist) + if direct_url is None: + return "pypi" + + vcs_info = direct_url.get("vcs_info") + if isinstance(vcs_info, dict): + commit = vcs_info.get("commit_id") or "" + if commit: + return f"git+{commit[:SHORT_COMMIT_LENGTH]}" + return "git" + + return "local" + + +def _contributed_modules( + entry_points: list[importlib.metadata.EntryPoint], distribution: str +) -> str: + """Count the forensic modules a plugin package contributes. + + Entry points are resolved the way MVT resolves them when it loads + modules, but a broken entry point is reported instead of raising: listing + the installed plugins must work even when one of them is faulty. + """ + count = 0 + broken = False + + for entry_point in entry_points: + if _entry_point_distribution(entry_point) != distribution: + continue + try: + loaded = entry_point.load() + if callable(loaded) and not isinstance(loaded, type): + loaded = loaded() + count += sum( + 1 + for module in loaded + if isinstance(module, type) and issubclass(module, MVTModule) + ) + except (Exception, SystemExit) as exc: + log.debug( + "Unable to load the modules of entry point %s (%s): %s", + entry_point.name, + entry_point.value, + exc, + ) + broken = True + + if broken: + return f"{count} (error)" if count else "error" + + return str(count) + + +def _contributed_commands( + entry_points: list[importlib.metadata.EntryPoint], distribution: str +) -> str: + names = { + entry_point.name + for entry_point in entry_points + if _entry_point_distribution(entry_point) == distribution + } + + return ", ".join(sorted(names)) if names else "-" + + +@click.group("plugins", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS) +def plugins() -> None: + pass + + +@plugins.command("list", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS_LIST) +def list_plugins() -> None: + distributions = installed_plugin_distributions() + if not distributions: + click.echo("No MVT plugins are installed.") + return + + module_entry_points = _entry_points(MODULES_ENTRY_POINT_GROUP) + command_entry_points = [] + for group in CLI_PLUGIN_GROUPS: + command_entry_points.extend(_entry_points(group)) + + table = Table(title="Installed MVT plugins") + table.add_column("Name", style="bold") + table.add_column("Version") + table.add_column("Origin") + table.add_column("Modules", justify="right") + table.add_column("Commands") + + for dist in distributions: + name = dist.name + table.add_row( + name, + _distribution_version(dist), + _distribution_origin(dist), + _contributed_modules(module_entry_points, name), + _contributed_commands(command_entry_points, name), + ) + + Console().print(table) + + +@plugins.command( + "check-updates", + context_settings=CONTEXT_SETTINGS, + help=HELP_MSG_PLUGINS_CHECK_UPDATES, + short_help="Check the installed plugins for updates", +) +def check_plugin_updates() -> None: + if not settings.NETWORK_ACCESS_ALLOWED: + click.echo( + "Network access is disabled, cannot check for plugin updates. " + "Enable NETWORK_ACCESS_ALLOWED in the MVT configuration to check." + ) + return + + if not installed_plugin_distributions(): + click.echo("No MVT plugins are installed.") + return + + findings = PluginUpdates().check() + if not findings: + click.echo("All plugins are up to date.") + return + + click.echo("Plugin updates available:") + for finding in findings: + click.echo(f" {finding['name']} {finding['installed']} → {finding['latest']}") + click.echo(f" Upgrade with: {finding['upgrade_command']}") + + click.echo( + "\nMVT does not install plugin updates. Run the command above when you " + "decide to upgrade." + ) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index aab53e6..1242941 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -19,7 +19,12 @@ from .alerts import AlertLevel, AlertStore from .config import settings from .indicators import Indicators from .module import EncryptedBackupError, MVTModule, run_module, save_timeline -from .module_loader import module_supports_command +from .module_loader import ( + ModuleOrigin, + get_module_logger, + get_module_origin, + module_supports_command, +) from .module_types import ModuleTimeline, URLResult from .utils import ( CustomJSONEncoder, @@ -67,6 +72,10 @@ class Command: # down a password to decrypt a backup or flags which are need by some modules. self.module_options = module_options if module_options else {} + # This dictionary maps the modules which were replaced by a module + # declaring `replaces` to the module which took their place. + self.module_replacements: dict[type[MVTModule], type[MVTModule]] = {} + # This list will contain all executed modules. # We can use this to reference e.g. self.executed[0].results. self.executed: list[MVTModule] = [] @@ -210,10 +219,34 @@ class Command: for file in generate_hashes_from_path(self.target_path, self.log): self.hash_values.append(file) + @staticmethod + def _modules_by_origin( + modules: list[type[MVTModule]], + ) -> dict[ModuleOrigin, list[str]]: + origins: dict[ModuleOrigin, list[str]] = {} + for module in modules: + origins.setdefault(get_module_origin(module), []).append(module.__name__) + return origins + def list_modules(self) -> None: self.log.info("Following is the list of available %s modules:", self.name) - for module in self._available_modules(): - self.log.info(" - %s", module.__name__) + for origin, module_names in self._modules_by_origin( + self._available_modules() + ).items(): + self.log.info( + " - Modules from %s: %s", origin.label, ", ".join(module_names) + ) + + def _log_loaded_modules(self, modules: list[type[MVTModule]]) -> None: + """Record the loaded modules and their origins for auditability.""" + for origin, module_names in self._modules_by_origin(modules).items(): + self.log.info( + "Loaded %d %s modules from %s: %s", + len(module_names), + self.name, + origin.label, + ", ".join(module_names), + ) def _available_modules(self) -> list[type[MVTModule]]: modules = list(self.modules) @@ -228,7 +261,174 @@ class Command: if module not in deduplicated: deduplicated.append(module) - return deduplicated + available = self._apply_replacements(deduplicated) + self._warn_about_slug_collisions(available) + + return available + + def _warn_about_slug_collisions(self, modules: list[type[MVTModule]]) -> None: + """Report modules writing their results to the same file. + + Results are stored in a file named after the module slug, so two + modules sharing one silently overwrite each other. Replacements are + already resolved here, so a module deliberately taking over the slug + of the module it replaces is not reported: the module it replaced is + no longer part of the run. + """ + modules_by_slug: dict[str, type[MVTModule]] = {} + for module in modules: + slug = module.get_slug() + first = modules_by_slug.setdefault(slug, module) + if first is module: + continue + + self.log.warning( + "Modules %s from %s and %s from %s both use the slug %s. If " + "both run, whichever runs last overwrites the results of the " + "other in %s.json.", + first.__name__, + get_module_origin(first).label, + module.__name__, + get_module_origin(module).label, + slug, + slug, + ) + + def _declared_replacements( + self, modules: list[type[MVTModule]] + ) -> dict[type[MVTModule], type[MVTModule]]: + """Return the replacements declared by the given modules.""" + replacements: dict[type[MVTModule], type[MVTModule]] = {} + for module in modules: + replaced = module.replaces + if replaced is None or replaced is module: + continue + + if not module.enabled: + # Replacing a module must not disable it, as a disabled + # replacement never runs in its place. + self.log.debug( + "Module %s is disabled and does not replace module %s.", + module.__name__, + replaced.__name__, + ) + continue + + if replaced not in modules: + # A module can support several commands while the module it + # replaces is only available in some of them. + self.log.debug( + "Module %s replaces module %s, which is not available " + "for the %s command.", + module.__name__, + replaced.__name__, + self.name, + ) + continue + + if replaced in replacements: + self.log.warning( + "Modules %s and %s both replace module %s. Both of them " + "will run, %s will not, and modules depending on %s will " + "use the results of %s. Replacements which share the slug " + "of %s overwrite each other's results file.", + replacements[replaced].__name__, + module.__name__, + replaced.__name__, + replaced.__name__, + replaced.__name__, + replacements[replaced].__name__, + replaced.__name__, + ) + continue + + replacements[replaced] = module + + return replacements + + def _drop_replacement_cycles( + self, replacements: dict[type[MVTModule], type[MVTModule]] + ) -> None: + """Undo the replacements between modules which replace each other.""" + cyclic: set[type[MVTModule]] = set() + for replaced in replacements: + walked: list[type[MVTModule]] = [] + module = replaced + while module in replacements and module not in cyclic: + if module in walked: + cyclic.update(walked[walked.index(module) :]) + break + walked.append(module) + module = replacements[module] + + if not cyclic: + return + + self.log.warning( + "Modules %s replace each other in a cycle. None of them replaces " + "anything and all of them will run.", + ", ".join(sorted(module.__name__ for module in cyclic)), + ) + for module in cyclic: + replacements.pop(module, None) + + def _log_replacement( + self, + replaced: type[MVTModule], + module: type[MVTModule], + replacement: type[MVTModule], + ) -> None: + """Report an applied replacement, and any problem with it.""" + if not issubclass(module, replaced): + self.log.warning( + "Module %s replaces module %s but is not a subclass of it. " + "Its results might not be compatible with what modules " + "depending on %s expect.", + module.__name__, + replaced.__name__, + replaced.__name__, + ) + + if replaced in module.dependencies: + self.log.warning( + "Module %s depends on module %s, which it also replaces. The " + "dependency cannot be satisfied: a replacement has to produce " + "that data itself.", + module.__name__, + replaced.__name__, + ) + + self.log.info( + "Module %s from %s replaces module %s from %s.", + replacement.__name__, + get_module_origin(replacement).label, + replaced.__name__, + get_module_origin(replaced).label, + ) + + def _apply_replacements( + self, modules: list[type[MVTModule]] + ) -> list[type[MVTModule]]: + """Drop the modules superseded by a module declaring `replaces`.""" + declared = self._declared_replacements(modules) + self._drop_replacement_cycles(declared) + + replacements: dict[type[MVTModule], type[MVTModule]] = {} + for replaced, module in declared.items(): + # Follow chains of replacements, so that a dependency on a + # replaced module always resolves to a module which is part of + # the run. + replacement = module + while replacement in declared: + replacement = declared[replacement] + + # Only the replacements which are applied are reported, so that + # the record matches the modules which actually run. + self._log_replacement(replaced, module, replacement) + replacements[replaced] = replacement + + self.module_replacements = replacements + return [module for module in modules if module not in replacements] def init(self) -> None: raise NotImplementedError @@ -288,42 +488,196 @@ class Command: console.print("") console.print(panel) + def _module_dependencies( + self, module: type[MVTModule] + ) -> list[tuple[type[MVTModule], type[MVTModule]]]: + """Return the (declared, resolved) dependencies of a module. + + A dependency on a module which was replaced is resolved to the module + which took its place. A module which replaces one of its own + dependencies is not made to depend on itself, while a module which + declares itself as a dependency is left alone and still fails the + circular dependency check. + """ + dependencies = [] + for dependency in module.dependencies: + resolved = self.module_replacements.get(dependency, dependency) + if resolved is module and dependency is not module: + continue + dependencies.append((dependency, resolved)) + + return dependencies + + @staticmethod + def _dependency_name( + declared: type[MVTModule], resolved: type[MVTModule] + ) -> str: + """Return how a dependency is named in the messages about it. + + A dependency is named as the module which declared it wrote it, and, + when that module was replaced, as the module which runs in its place. + """ + if declared is resolved: + return declared.__name__ + + return f"{resolved.__name__} (replacing module {declared.__name__})" + + def _selected_modules( + self, modules: list[type[MVTModule]] + ) -> Optional[list[type[MVTModule]]]: + """Return the modules explicitly requested, or all the enabled ones. + + Returns None when a module was requested by name and no module of + that name can be run. + """ + if not self.module_name: + return [module for module in modules if module.enabled] + + selected = [ + module for module in modules if module.__name__ == self.module_name + ] + + # A module replacing another one does not have to keep its name, so + # the name of a replaced module selects its replacement. + if not selected: + for replaced, replacement in self.module_replacements.items(): + if replaced.__name__ != self.module_name or replacement in selected: + continue + self.log.info( + "Module %s was replaced by module %s, which is run " + "in its place.", + replaced.__name__, + replacement.__name__, + ) + selected.append(replacement) + + if not selected: + self.log.warning( + "No module named %s is available for the %s command. " + "No modules will be run.", + self.module_name, + self.name, + ) + return None + + return selected + + def _skipped_modules( + self, + required: list[type[MVTModule]], + module_indexes: dict[type[MVTModule], int], + ) -> dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]]: + """Return the modules to drop because a dependency is unavailable. + + A module declaring a dependency this command cannot provide is unable + to run, and so is every module depending on it. Dropping only those + keeps a single wrong declaration - in a module scoped to several + commands, for example - from silencing an entire analysis. + + The returned mapping gives, for each skipped module, the module which + is missing a dependency and the dependency it is missing. + """ + skipped: dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]] = {} + # Skipping the one module a run was asked for leaves nothing to run, + # which the caller reports instead. + remainder = ( + "" if self.module_name else " The rest of the analysis will still run." + ) + + # Skipping one module can skip the modules depending on it, which the + # pass over the module list may already have gone past, so repeat the + # pass until nothing changes. + changed = True + while changed: + changed = False + for module in required: + if module in skipped: + continue + + for declared, dependency in self._module_dependencies(module): + if dependency not in module_indexes: + # A replaced dependency always resolves to a module of + # this command, so an unavailable one is always the + # module class the author declared. + skipped[module] = (module, declared) + changed = True + self.log.warning( + "Module %s will be SKIPPED: it depends on module " + "%s, which is not available in this command.%s", + module.__name__, + declared.__name__, + remainder, + ) + break + + if dependency in skipped: + root, missing = skipped[dependency] + skipped[module] = (root, missing) + changed = True + if dependency is root: + self.log.warning( + "Module %s will be SKIPPED: it depends on " + "module %s, itself skipped for depending on " + "unavailable module %s.%s", + module.__name__, + self._dependency_name(declared, dependency), + missing.__name__, + remainder, + ) + else: + self.log.warning( + "Module %s will be SKIPPED: it depends on " + "skipped module %s, in a chain starting at " + "module %s, which depends on unavailable " + "module %s.%s", + module.__name__, + self._dependency_name(declared, dependency), + root.__name__, + missing.__name__, + remainder, + ) + break + + return skipped + def _ordered_modules(self) -> Optional[list[type[MVTModule]]]: """Return enabled modules in stable topological order.""" modules = self._available_modules() module_indexes = {module: index for index, module in enumerate(modules)} - if self.module_name: - selected = [ - module for module in modules if module.__name__ == self.module_name - ] - else: - selected = [module for module in modules if module.enabled] + selected = self._selected_modules(modules) + if selected is None: + return None - required = set(selected) + required: set[type[MVTModule]] = set() pending = list(selected) while pending: module = pending.pop() - for dependency in module.dependencies: - if dependency not in module_indexes: - self.log.warning( - "Module %s depends on unavailable module %s. " - "No modules will be run.", - module.__name__, - dependency.__name__, - ) - return None - if dependency not in required: - required.add(dependency) + if module in required: + continue + required.add(module) + for _, dependency in self._module_dependencies(module): + # Unavailable dependencies are reported by _skipped_modules(). + if dependency in module_indexes: pending.append(dependency) + ordered_required = sorted(required, key=lambda module: module_indexes[module]) + skipped = self._skipped_modules(ordered_required, module_indexes) + runnable = [module for module in ordered_required if module not in skipped] + if skipped and not runnable: + self.log.warning( + "Every selected module was skipped for an unavailable " + "dependency. No modules will be run." + ) + return None + dependents: dict[type[MVTModule], list[type[MVTModule]]] = { - module: [] for module in required + module: [] for module in runnable } - indegree = {module: 0 for module in required} - for module in required: - for dependency in module.dependencies: - if dependency not in required: + indegree = {module: 0 for module in runnable} + for module in runnable: + for _, dependency in self._module_dependencies(module): + if dependency not in indegree: continue dependents[dependency].append(module) indegree[module] += 1 @@ -342,7 +696,7 @@ class Command: if indegree[dependent] == 0: heappush(ready, (module_indexes[dependent], dependent)) - if len(ordered) != len(required): + if len(ordered) != len(runnable): cyclic_modules = sorted( (module.__name__ for module, count in indegree.items() if count > 0) ) @@ -360,6 +714,8 @@ class Command: if ordered_modules is None: return + self._log_loaded_modules(ordered_modules) + try: self.init() except NotImplementedError: @@ -368,7 +724,7 @@ class Command: executed_by_type: dict[type[MVTModule], MVTModule] = {} for module in ordered_modules: - module_logger = logging.getLogger(module.__module__) + module_logger = get_module_logger(module) m = module( target_path=self.target_path, @@ -376,9 +732,12 @@ class Command: module_options=self.module_options, log=module_logger, ) + # Dependencies are keyed by the module class they declare, even + # when it was replaced, so that a module asking for the results + # of a replaced module receives those of its replacement. m.dependency_modules = { - dependency: executed_by_type[dependency] - for dependency in module.dependencies + dependency: executed_by_type[resolved] + for dependency, resolved in self._module_dependencies(module) } if self.iocs.total_ioc_count: diff --git a/src/mvt/common/completion.py b/src/mvt/common/completion.py index 6466a6d..0b72b4e 100644 --- a/src/mvt/common/completion.py +++ b/src/mvt/common/completion.py @@ -9,34 +9,83 @@ import shlex import click from click.shell_completion import get_completion_class +from .help import HELP_MSG_COMPLETION + SUPPORTED_SHELLS = ("bash", "zsh", "fish") +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +COMPLETION_INSTRUCTIONS = """Shell completion for mvt, mvt-ios and mvt-android -def completion_instructions(program_name: str) -> str: - return f"""Shell completion for {program_name} - -Print a completion script: - {program_name} completion bash > ~/.{program_name}-complete.bash - {program_name} completion zsh > ~/.{program_name}-complete.zsh - mkdir -p ~/.config/fish/completions - {program_name} completion fish > ~/.config/fish/completions/{program_name}.fish +Print one completion script covering the three commands: + mvt completion bash > ~/.mvt-complete.bash + mvt completion zsh > ~/.mvt-complete.zsh + mkdir -p ~/.config/fish/conf.d + mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish Load the generated Bash script from ~/.bashrc: - [ -f ~/.{program_name}-complete.bash ] && . ~/.{program_name}-complete.bash + [ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash Load the generated Zsh script from ~/.zshrc: - [ -f ~/.{program_name}-complete.zsh ] && . ~/.{program_name}-complete.zsh + [ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh -Fish loads completion files from ~/.config/fish/completions automatically. +Fish loads the files in ~/.config/fish/conf.d automatically. -To write these files and update Bash/Zsh shell configuration automatically: - {program_name} completion bash --install - {program_name} completion zsh --install - {program_name} completion fish --install +To write these files and update the Bash/Zsh shell configuration automatically: + mvt completion bash --install + mvt completion zsh --install + mvt completion fish --install """ +def _mvt_programs() -> list[tuple[str, click.Command]]: + """Return the console script name and CLI group of every MVT program. + + The three CLIs are imported here rather than at module level: mvt.cli + imports this module while it is being defined, and generating a completion + script should not make the start-up of `mvt` import the platform CLIs. + """ + from mvt.android.cli import cli as android_cli + from mvt.cli import cli as mvt_cli + from mvt.ios.cli import cli as ios_cli + + return [("mvt", mvt_cli), ("mvt-ios", ios_cli), ("mvt-android", android_cli)] + + +@click.command( + "completion", + context_settings=CONTEXT_SETTINGS, + help=HELP_MSG_COMPLETION, + short_help="Generate or install shell completion", +) +@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) +@click.option( + "--install", + is_flag=True, + help="Write completion files and update shell configuration.", +) +def completion(shell, install): + if shell is None: + if install: + raise click.UsageError("A shell is required when using --install.") + click.echo(COMPLETION_INSTRUCTIONS) + return + + if install: + script_path = install_completion_script(shell) + click.echo( + f"Installed {shell} completion for mvt, mvt-ios and mvt-android " + f"to {script_path}" + ) + if shell in ("bash", "zsh"): + click.echo(f"Updated ~/.{shell}rc") + else: + click.echo("Fish loads the files in ~/.config/fish/conf.d automatically.") + return + + click.echo(generate_mvt_completion_script(shell)) + + def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str: completion_class = get_completion_class(shell) if completion_class is None: @@ -46,41 +95,49 @@ def generate_completion_script(cli: click.Command, program_name: str, shell: str return completion_class(cli, {}, program_name, complete_var).source() -def install_completion_script( - cli: click.Command, - program_name: str, - shell: str, -) -> Path: - script = generate_completion_script(cli, program_name, shell) - script_path = _completion_script_path(program_name, shell) +def generate_mvt_completion_script(shell: str) -> str: + """Return one script completing every MVT command. + + Click names the completion function of each program after the program, so + the scripts of the three commands can simply be concatenated. + """ + scripts = [ + generate_completion_script(cli, program_name, shell).strip("\n") + for program_name, cli in _mvt_programs() + ] + return "\n\n".join(scripts) + + +def install_completion_script(shell: str) -> Path: + script = generate_mvt_completion_script(shell) + script_path = _completion_script_path(shell) script_path.parent.mkdir(parents=True, exist_ok=True) - script_path.write_text(script, encoding="utf-8") + script_path.write_text(f"{script}\n", encoding="utf-8") if shell in ("bash", "zsh"): - _install_shell_source_line(program_name, shell, script_path) + _install_shell_source_line(shell, script_path) return script_path -def _completion_script_path(program_name: str, shell: str) -> Path: +def _completion_script_path(shell: str) -> Path: home = Path.home() if shell == "fish": - return home / ".config" / "fish" / "completions" / f"{program_name}.fish" + # conf.d is sourced when the shell starts, unlike the completions + # folder, whose files fish loads on demand by command name. + return home / ".config" / "fish" / "conf.d" / "mvt-completion.fish" - return home / f".{program_name}-complete.{shell}" + return home / f".mvt-complete.{shell}" -def _install_shell_source_line(program_name: str, shell: str, script_path: Path) -> None: +def _install_shell_source_line(shell: str, script_path: Path) -> None: shell_config_path = Path.home() / f".{shell}rc" source_line = ( f"[ -f {shlex.quote(str(script_path))} ] && " f". {shlex.quote(str(script_path))}" ) - block = ( - f"# MVT shell completion for {program_name}\n" - f"{source_line}\n" - ) + block = f"# MVT shell completion\n{source_line}\n" if shell_config_path.exists(): shell_config = shell_config_path.read_text(encoding="utf-8") diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 5101f93..8b70f20 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -8,7 +8,7 @@ HELP_MSG_VERSION = "Show the currently installed version of MVT" HELP_MSG_OUTPUT = "Specify a path to a folder where you want to store JSON results" HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)" HELP_MSG_FAST = "Avoid running time/resource consuming features" -HELP_MSG_LIST_MODULES = "Print list of available modules and exit" +HELP_MSG_LIST_MODULES = "Print list of available modules and their source, then exit" HELP_MSG_MODULE = "Name of a single module you would like to run instead of all" HELP_MSG_LOAD_MODULE = ( "Load custom MVT module(s) from a Python file or folder " @@ -17,11 +17,22 @@ HELP_MSG_LOAD_MODULE = ( HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing" HELP_MSG_HASHES = "Generate hashes of all the files analyzed" HELP_MSG_VERBOSE = "Verbose mode" +HELP_MSG_VERBOSE_COMMAND = ( + "Verbose mode (kept for compatibility, pass --verbose before the command " + "name instead)" +) HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators" HELP_MSG_STIX2 = "Download public STIX2 indicators" HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check" HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check" -HELP_MSG_COMPLETION = "Generate or install shell completion" +HELP_MSG_COMPLETION = ( + "Generate or install shell completion for mvt, mvt-ios and mvt-android" +) +HELP_MSG_PLUGINS = "Inspect the installed MVT plugin packages" +HELP_MSG_PLUGINS_LIST = "List the installed plugins and what they contribute to MVT" +HELP_MSG_PLUGINS_CHECK_UPDATES = ( + "Check the installed plugins for updates without installing them" +) # IOS Specific HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup" diff --git a/src/mvt/common/logo.py b/src/mvt/common/logo.py index 048ee22..96836a9 100644 --- a/src/mvt/common/logo.py +++ b/src/mvt/common/logo.py @@ -8,10 +8,125 @@ import logging import requests from rich import print as rich_print -from .updates import IndicatorsUpdates, MVTUpdates +from .config import settings +from .updates import ( + IndicatorsUpdates, + MVTUpdates, + PluginUpdates, + installed_plugin_distributions, +) from .version import MVT_VERSION +def _check_version_updates(log: logging.Logger) -> None: + try: + mvt_updates = MVTUpdates() + latest_version = mvt_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for MVT updates.[/bold] " + "You may be working offline. Please update MVT regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check latest MVT version: %s", e) + else: + if latest_version: + rich_print( + f"\t[bold]Version {latest_version} is available! " + "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" + ) + + +def _check_indicator_updates(log: logging.Logger) -> None: + ioc_updates = IndicatorsUpdates() + + # Before proceeding, we check if we have downloaded an indicators index. + # If not, there's no point in proceeding with the updates check. + if ioc_updates.get_latest_update() == 0: + rich_print( + "\t[bold]You have not yet downloaded any indicators, check " + "the `download-iocs` command![/bold]" + ) + return + + # We only perform this check at a fixed frequency, in order to not + # overburden the user with too many lookups if the command is being run + # multiple times. + should_check, hours = ioc_updates.should_check() + if not should_check: + rich_print( + f"\tIndicators updates checked recently, next automatic check " + f"in {int(hours)} hours" + ) + return + + try: + ioc_to_update = ioc_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for indicator updates.[/bold] " + "You may be working offline. Please update MVT indicators regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check latest MVT indicators: %s", e) + else: + if ioc_to_update: + rich_print( + "\t[bold]There are updates to your indicators files! " + "Run the `download-iocs` command to update![/bold]" + ) + else: + rich_print("\tYour indicators files seem to be up to date.") + + +def _print_plugin_updates(findings: list) -> None: + if not findings: + return + + rich_print("\t[bold]Plugin updates available:[/bold]") + for finding in findings: + rich_print( + f"\t {finding['name']} {finding['installed']} → " + f"{finding['latest']} ({finding['upgrade_command']})" + ) + + +def _check_plugin_updates(log: logging.Logger) -> None: + if not settings.NETWORK_ACCESS_ALLOWED: + return + + # This runs on every command, so nothing here, including reading back what + # the latest check stored, may ever interrupt MVT. + try: + distributions = installed_plugin_distributions() + + # There is nothing to check when MVT was not extended with any plugin. + if not distributions: + return + + plugin_updates = PluginUpdates() + + # We only perform this check at a fixed frequency, in order to not + # overburden the user (and the plugin repositories) with too many + # lookups. In between checks we print the findings of the latest one, + # leaving out those which no longer apply to what is installed. + should_check, _ = plugin_updates.should_check() + if not should_check: + _print_plugin_updates(plugin_updates.current_findings(distributions)) + return + + findings = plugin_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for plugin updates.[/bold] " + "You may be working offline. Please update your MVT plugins regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check MVT plugin updates: %s", e) + else: + _print_plugin_updates(findings) + + def check_updates( disable_version_check: bool = False, disable_indicator_check: bool = False ) -> None: @@ -19,77 +134,25 @@ def check_updates( # First we check for MVT version updates. if not disable_version_check: - try: - mvt_updates = MVTUpdates() - latest_version = mvt_updates.check() - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): - rich_print( - "\t\t[bold]Note: Could not check for MVT updates.[/bold] " - "You may be working offline. Please update MVT regularly." - ) - except Exception as e: - log.error( - "Error encountered when trying to check latest MVT version: %s", e - ) - else: - if latest_version: - rich_print( - f"\t\t[bold]Version {latest_version} is available! " - "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" - ) + _check_version_updates(log) # Then we check for indicators files updates. if not disable_indicator_check: - ioc_updates = IndicatorsUpdates() + _check_indicator_updates(log) - # Before proceeding, we check if we have downloaded an indicators index. - # If not, there's no point in proceeding with the updates check. - if ioc_updates.get_latest_update() == 0: - rich_print( - "\t\t[bold]You have not yet downloaded any indicators, check " - "the `download-iocs` command![/bold]" - ) - return - - # We only perform this check at a fixed frequency, in order to not - # overburden the user with too many lookups if the command is being run - # multiple times. - should_check, hours = ioc_updates.should_check() - if not should_check: - rich_print( - f"\t\tIndicators updates checked recently, next automatic check " - f"in {int(hours)} hours" - ) - return - - try: - ioc_to_update = ioc_updates.check() - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): - rich_print( - "\t\t[bold]Note: Could not check for indicator updates.[/bold] " - "You may be working offline. Please update MVT indicators regularly." - ) - except Exception as e: - log.error( - "Error encountered when trying to check latest MVT indicators: %s", e - ) - else: - if ioc_to_update: - rich_print( - "\t\t[bold]There are updates to your indicators files! " - "Run the `download-iocs` command to update![/bold]" - ) - else: - rich_print("\t\tYour indicators files seem to be up to date.") + # Finally we check for updates to the installed plugin packages. MVT never + # installs an update itself, it only reports the command which does. + if not disable_version_check: + _check_plugin_updates(log) def logo( disable_version_check: bool = False, disable_indicator_check: bool = False ) -> None: rich_print("\n") - rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit") - rich_print("\t\thttps://mvt.re") - rich_print(f"\t\tVersion: {MVT_VERSION}") + rich_print("\t[bold]MVT - Mobile Verification Toolkit[/bold]\n") + rich_print("\thttps://mvt.re") + rich_print(f"\tVersion: {MVT_VERSION}\n") check_updates(disable_version_check, disable_indicator_check) diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index cd127a6..1257aa6 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -46,6 +46,11 @@ class MVTModule: slug: Optional[str] = None dependencies: Sequence[type["MVTModule"]] = () supported_commands: Sequence[tuple[str, str]] = () + # A custom module can name a module class it supersedes, usually a + # built-in one. When both are available to a command, the named module is + # dropped from the run and this module takes its place, including in the + # dependencies of any other module. + replaces: Optional[type["MVTModule"]] = None def __init__( self, diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index da94826..51b3836 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -4,18 +4,31 @@ # https://license.mvt.re/1.1/ import hashlib +import importlib.metadata import importlib.util import inspect +import json import logging import os +import re import sys +from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from types import ModuleType from typing import Iterable, Optional +from .cli_plugins import CUSTOM_COMMAND_MODULE_PREFIX from .module import MVTModule +from .version import MVT_VERSION MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES" +MODULES_ENTRY_POINT_GROUP = "mvt.modules" +EXTERNAL_LOGGER_NAMESPACE = "mvt.ext" +PLUGIN_PACKAGE_PREFIX = "mvt_plugin_" +_ORIGIN_ATTRIBUTE = "_mvt_module_origin" +_PATH_MODULE_PREFIX = "_mvt_custom_module_" +_LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$") log = logging.getLogger(__name__) @@ -23,15 +36,109 @@ class CustomModuleLoadError(Exception): pass +@dataclass(frozen=True) +class ModuleOrigin: + """Describes where a loaded module came from, for auditability. + + ``kind`` is one of ``builtin`` (shipped with MVT), ``package`` (loaded + from an installed package) or ``path`` (loaded from a file passed with + ``--load-module`` or the environment variable). + """ + + kind: str + name: str + version: Optional[str] = None + commit: Optional[str] = None + file_sha256: Optional[str] = None + + @property + def label(self) -> str: + label = self.name + if self.version: + label += f"@{self.version}" + label = f"'{label}'" + if self.commit: + label += f" (commit {self.commit})" + if self.file_sha256: + label += f" (sha256: {self.file_sha256})" + return label + + def _module_name_for_path(path: Path) -> str: digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] - return f"_mvt_custom_module_{path.stem}_{digest}" + return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}" + + +def _is_builtin_logger_name(name: str) -> bool: + return name == "mvt" or name.startswith("mvt.") + + +def _loaded_file_stem(name: str, prefix: str) -> str: + """Recover a loaded file's name from the import name MVT gave it.""" + return _LOADED_FILE_DIGEST.sub("", name[len(prefix) :]) + + +def _external_logger_name(name: str) -> str: + """Return the "mvt.ext" logger name external code logs under.""" + top_level, separator, rest = name.partition(".") + if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len( + PLUGIN_PACKAGE_PREFIX + ): + name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest + return f"{EXTERNAL_LOGGER_NAMESPACE}.{name}" + + +def get_module_logger(module_class: type[MVTModule]) -> logging.Logger: + """Return the logger a module's records should be emitted through. + + Modules loaded from installed packages or file paths live outside the + "mvt" logger hierarchy, so their records would never reach the handlers + attached to the "mvt" logger and instead fall through to + logging.lastResort (which prints bare messages and drops anything below + WARNING). Their loggers are parented under the "mvt.ext" namespace, + keeping external module names from colliding with MVT's own logger + tree. File-path modules are named after their file instead of the + mangled internal import name, and packages following the recommended + "mvt_plugin_" naming convention log under "mvt.ext.". + """ + name = module_class.__module__ + if _is_builtin_logger_name(name): + return logging.getLogger(name) + + if name.startswith(_PATH_MODULE_PREFIX): + file_name = Path(get_module_origin(module_class).name).stem + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{file_name}") + + return logging.getLogger(_external_logger_name(name)) + + +def get_plugin_logger(name: str) -> logging.Logger: + """Return a general logger for use in custom MVT plugins. + + Call it with ``__name__``. The logger sits under "mvt.ext". That is + where get_module_logger() puts module classes. A file loaded with + --load-module or --load-command is named after the file. + """ + if _is_builtin_logger_name(name): + return logging.getLogger(name) + + # A file loaded with --load-module or --load-command is imported under a + # mangled name. Log it under the file it came from. get_module_logger() + # does the same for the module classes such a file defines. + for prefix in (_PATH_MODULE_PREFIX, CUSTOM_COMMAND_MODULE_PREFIX): + if name.startswith(prefix): + stem = _loaded_file_stem(name, prefix) + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{stem}") + + return logging.getLogger(_external_logger_name(name)) def _iter_module_files(path: Path) -> Iterable[Path]: if path.is_file(): if path.suffix != ".py": - raise CustomModuleLoadError(f"Custom module file is not a Python file: {path}") + raise CustomModuleLoadError( + f"Custom module file is not a Python file: {path}" + ) yield path return @@ -59,7 +166,9 @@ def _load_python_file(path: Path) -> ModuleType: try: spec.loader.exec_module(module) except Exception as exc: - raise CustomModuleLoadError(f"Unable to import custom module {path}: {exc}") from exc + raise CustomModuleLoadError( + f"Unable to import custom module {path}: {exc}" + ) from exc return module @@ -84,17 +193,171 @@ def load_custom_modules_from_path(path: str) -> list[type[MVTModule]]: resolved_path = Path(path).expanduser().resolve() for module_file in _iter_module_files(resolved_path): + file_sha256 = hashlib.sha256(module_file.read_bytes()).hexdigest() loaded_module = _load_python_file(module_file) + origin = ModuleOrigin( + kind="path", name=str(module_file), file_sha256=file_sha256 + ) for module_class in discover_mvt_modules(loaded_module): key = (str(module_file), module_class.__qualname__) if key in seen: continue seen.add(key) + setattr(module_class, _ORIGIN_ATTRIBUTE, origin) custom_modules.append(module_class) return custom_modules +def _module_key(module_class: type[MVTModule]) -> tuple[str, str]: + try: + source = str(Path(inspect.getfile(module_class)).resolve()) + except (OSError, TypeError): + source = module_class.__module__ + return (source, module_class.__qualname__) + + +def distribution_direct_url(dist: importlib.metadata.Distribution) -> Optional[dict]: + """Return the PEP 610 direct URL metadata of a distribution, if recorded. + + Packages installed from an index have no direct URL metadata, while + packages installed directly from a repository or from a local folder + record where they were installed from in ``direct_url.json``. + """ + try: + direct_url_text = dist.read_text("direct_url.json") + if not direct_url_text: + return None + direct_url = json.loads(direct_url_text) + return direct_url if isinstance(direct_url, dict) else None + except Exception: + return None + + +def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]: + """Return the VCS commit a distribution was installed from, if recorded. + + Packages installed directly from a repository (``pip install git+...``) + record the commit in ``direct_url.json`` (PEP 610). + """ + vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info") + if not isinstance(vcs_info, dict): + return None + + commit = vcs_info.get("commit_id") + return commit if isinstance(commit, str) else None + + +def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin: + name = entry_point.name + version = None + commit = None + # Manually constructed entry points have no associated distribution. + dist = getattr(entry_point, "dist", None) + if dist is not None: + try: + name = dist.name or name + version = dist.version + except Exception: + pass + commit = _distribution_commit(dist) + return ModuleOrigin(kind="package", name=name, version=version, commit=commit) + + +@lru_cache(maxsize=1) +def _packages_distributions() -> dict[str, list[str]]: + try: + return dict(importlib.metadata.packages_distributions()) + except Exception: + return {} + + +def get_module_origin(module_class: type[MVTModule]) -> ModuleOrigin: + """Return the origin of a module class for auditing purposes.""" + origin = module_class.__dict__.get(_ORIGIN_ATTRIBUTE) + if isinstance(origin, ModuleOrigin): + return origin + + top_level = module_class.__module__.partition(".")[0] + if top_level == "mvt": + return ModuleOrigin(kind="builtin", name="mvt", version=MVT_VERSION) + + distributions = _packages_distributions().get(top_level) + if distributions: + name = distributions[0] + version = None + commit = None + try: + dist = importlib.metadata.distribution(name) + version = dist.version + commit = _distribution_commit(dist) + except Exception: + pass + return ModuleOrigin(kind="package", name=name, version=version, commit=commit) + + try: + source = str(Path(inspect.getfile(module_class)).resolve()) + except (OSError, TypeError): + source = module_class.__module__ + return ModuleOrigin(kind="path", name=source) + + +def load_installed_modules() -> list[type[MVTModule]]: + """Load MVT modules registered by installed packages. + + Packages register modules in the ``mvt.modules`` entry-point group. Each + entry point must resolve to an iterable of MVTModule subclasses, or to a + callable which returns one. A broken entry point is skipped with a + warning so that a faulty plugin package cannot break MVT. + """ + try: + entry_points = importlib.metadata.entry_points(group=MODULES_ENTRY_POINT_GROUP) + except Exception as exc: + log.warning( + "Unable to discover installed module packages in entry-point group %s: %s", + MODULES_ENTRY_POINT_GROUP, + exc, + ) + return [] + + installed_modules: list[type[MVTModule]] = [] + ordered_entry_points = sorted( + entry_points, key=lambda entry_point: (entry_point.name, entry_point.value) + ) + for entry_point in ordered_entry_points: + try: + loaded = entry_point.load() + if callable(loaded) and not isinstance(loaded, type): + loaded = loaded() + module_classes = list(loaded) + except (Exception, SystemExit) as exc: + log.warning( + "Unable to load modules from entry point %s (%s): %s", + entry_point.name, + entry_point.value, + exc, + ) + continue + + origin = _entry_point_origin(entry_point) + for module_class in module_classes: + if not ( + isinstance(module_class, type) and issubclass(module_class, MVTModule) + ): + log.warning( + "Entry point %s (%s) provided %r which is not an " + "MVTModule subclass", + entry_point.name, + entry_point.value, + module_class, + ) + continue + setattr(module_class, _ORIGIN_ATTRIBUTE, origin) + installed_modules.append(module_class) + + return installed_modules + + def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTModule]]: search_paths: list[str] = [] env_path = os.environ.get(MVT_CUSTOM_MODULES_ENV) @@ -105,10 +368,17 @@ def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTM custom_modules: list[type[MVTModule]] = [] seen: set[tuple[str, str]] = set() + + for module_class in load_installed_modules(): + key = _module_key(module_class) + if key in seen: + continue + seen.add(key) + custom_modules.append(module_class) + for path in search_paths: for module_class in load_custom_modules_from_path(path): - source = Path(inspect.getfile(module_class)).resolve() - key = (str(source), module_class.__qualname__) + key = _module_key(module_class) if key in seen: continue seen.add(key) @@ -131,4 +401,14 @@ def module_supports_command( ) return False - return (platform, command) in {tuple(entry) for entry in supported_commands} + pairs = {tuple(entry) for entry in supported_commands} + if (platform, command) in pairs: + return True + + # A module which implements check_indicators() is re-checked by check-iocs + # for its platform. It does not need to declare the check-iocs pair. + return ( + command == "check-iocs" + and platform in {entry[0] for entry in pairs if entry} + and module_class.check_indicators is not MVTModule.check_indicators + ) diff --git a/src/mvt/common/plugin_config.py b/src/mvt/common/plugin_config.py new file mode 100644 index 0000000..f5e2099 --- /dev/null +++ b/src/mvt/common/plugin_config.py @@ -0,0 +1,299 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import contextlib +import json +import os +import re +import tempfile +from typing import Any, ClassVar, Dict, List, Tuple, Type, TypeVar + +import yaml +from appdirs import user_config_dir, user_data_dir +from pydantic import ValidationError +from pydantic_settings import ( + BaseSettings, + EnvSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +PLUGIN_CONFIG_FOLDER_NAME = "plugins" +# Not "plugins": on macOS the configuration and data folders are the same +# directory, and that name already holds the settings files. +PLUGIN_DATA_FOLDER_NAME = "plugin-data" +PLUGIN_ENV_PREFIX = "MVT_PLUGIN_" + +PLUGIN_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]*") + +PluginSettingsType = TypeVar("PluginSettingsType", bound="MVTPluginSettings") + + +class PluginConfigLoadError(Exception): + pass + + +def validate_plugin_name(plugin_name: Any) -> str: + """ + Check that a plugin name is safe to use in a file name and an env variable. + + :param plugin_name: Name to validate. + :returns: The validated plugin name. + """ + if not isinstance(plugin_name, str): + raise TypeError( + f"Plugin name must be a string, not {type(plugin_name).__name__}" + ) + if not PLUGIN_NAME_PATTERN.fullmatch(plugin_name): + raise ValueError( + f"Invalid plugin name {plugin_name!r}: plugin names must start with a " + "lowercase letter or a digit and may only contain lowercase letters, " + "digits and dashes" + ) + return plugin_name + + +def plugin_config_folder() -> str: + """ + Return the folder where plugins store their configuration files. + + The path is resolved on every call so it always reflects the current + environment. + """ + return os.path.join(user_config_dir("mvt"), PLUGIN_CONFIG_FOLDER_NAME) + + +def plugin_config_path(plugin_name: str) -> str: + """ + Return the path of the configuration file of a given plugin. + + :param plugin_name: Name of the plugin. + """ + return os.path.join( + plugin_config_folder(), f"{validate_plugin_name(plugin_name)}.yaml" + ) + + +def plugin_data_folder(plugin_name: str) -> str: + """ + Return the folder where a given plugin stores its data, creating it. + + Plugins should keep whatever they persist, such as caches or downloaded + artifacts, in this folder. It is created with owner-only permissions. The + path is resolved on every call so it always reflects the current + environment. A plugin with a settings class calls + `MVTPluginSettings.data_folder()` instead, which passes `plugin_name` here. + + :param plugin_name: Name of the plugin. + :returns: The path of the data folder of the plugin. + """ + # Validate the name before anything is created, so an unsafe name cannot + # leave a folder behind. + name = validate_plugin_name(plugin_name) + + # makedirs() applies its mode only to the last folder of the path, so + # MVT's own data folder keeps the default permissions while the two + # plugin folders are private. + data_folder = os.path.join(user_data_dir("mvt"), PLUGIN_DATA_FOLDER_NAME) + os.makedirs(data_folder, mode=0o700, exist_ok=True) + + folder = os.path.join(data_folder, name) + os.makedirs(folder, mode=0o700, exist_ok=True) + return folder + + +def plugin_env_prefix(plugin_name: str) -> str: + """ + Return the environment variable prefix used by a given plugin. + + Dashes are replaced by underscores. Plugin names cannot contain underscores, + so two different plugin names never share an environment namespace. + + :param plugin_name: Name of the plugin. + """ + name = validate_plugin_name(plugin_name).upper().replace("-", "_") + return f"{PLUGIN_ENV_PREFIX}{name}_" + + +def _settings_plugin_name(settings_cls: Type[BaseSettings]) -> str: + plugin_name = getattr(settings_cls, "plugin_name", None) + if plugin_name is None: + raise TypeError( + f"{settings_cls.__name__} must set a 'plugin_name' class attribute to " + "namespace its configuration file and environment variables" + ) + return validate_plugin_name(plugin_name) + + +def _plugin_yaml_source( + settings_cls: Type[BaseSettings], config_path: str +) -> YamlConfigSettingsSource: + """ + Build the YAML settings source of a plugin, reporting unusable files. + + A missing file is not an error, but a file which cannot be parsed or which + does not hold a mapping of setting names is reported with its path. + """ + try: + return YamlConfigSettingsSource(settings_cls, config_path) + except yaml.YAMLError as exc: + raise PluginConfigLoadError( + f"Invalid plugin configuration file {config_path}: {exc}" + ) from exc + except (TypeError, ValueError) as exc: + raise PluginConfigLoadError( + f"Invalid plugin configuration file {config_path}: the file must " + "contain a mapping of setting names to values" + ) from exc + + +class MVTPluginSettings(BaseSettings): + """ + Base class for plugin-namespaced settings. + + Subclass with typed fields and set `plugin_name`. Values resolve from + constructor arguments, then environment variables (MVT_PLUGIN__*), + then the plugin's YAML file (~/.config/mvt/plugins/.yaml), then field + defaults. + + Plugins must not store their settings in MVT's own configuration file: MVT + rewrites it with the fields it knows about, dropping anything else. + + `data_folder()` returns the folder the plugin keeps its data in. + """ + + model_config = SettingsConfigDict(extra="ignore") + + plugin_name: ClassVar[str] + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + plugin_name = _settings_plugin_name(cls) + # Namespace the environment variables of this plugin. Each pydantic + # model gets its own configuration dictionary, so this does not leak + # into other plugins. + cls.model_config["env_prefix"] = plugin_env_prefix(plugin_name) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + config_path = plugin_config_path(_settings_plugin_name(settings_cls)) + yaml_source = _plugin_yaml_source(settings_cls, config_path) + # Explicit arguments take precedence over environment variables, which + # in turn take precedence over the configuration file. + return (init_settings, env_settings, yaml_source) + + @classmethod + def load(cls: Type[PluginSettingsType]) -> PluginSettingsType: + """ + Load the settings of the plugin. + + A missing configuration file is not an error: the settings then come + from the environment and from the field defaults. + """ + return cls() + + @classmethod + def data_folder(cls) -> str: + """ + Return the data folder of the plugin, creating it. + + The folder is the one plugin_data_folder() returns for `plugin_name`, + so a plugin with a settings class does not repeat its name. + """ + return plugin_data_folder(_settings_plugin_name(cls)) + + def _environment_values(self) -> Dict[str, Any]: + """ + Return the settings values currently supplied by the environment. + + Values are validated by the model, so they can be compared with the + values held by this instance. + """ + settings_cls = type(self) + raw_values = EnvSettingsSource(settings_cls)() + names = [name for name in raw_values if name in settings_cls.model_fields] + if not names: + return {} + + # Fall back on the current values for the fields the environment does + # not set, so that required fields do not fail validation here. + current_values = json.loads(self.model_dump_json()) + try: + from_environment = settings_cls.model_validate( + {**current_values, **raw_values} + ) + except ValidationError: + # An environment variable which the model cannot validate must not + # stop the other environment values from being recognised, or a + # credential would be written to the configuration file. + return self._environment_values_by_field(current_values, raw_values, names) + return {name: getattr(from_environment, name) for name in names} + + def _environment_values_by_field( + self, + current_values: Dict[str, Any], + raw_values: Dict[str, Any], + names: List[str], + ) -> Dict[str, Any]: + """ + Validate each environment value on its own, skipping unusable ones. + + :param current_values: Serialized values held by this instance. + :param raw_values: Values supplied by the environment. + :param names: Names of the fields set by the environment. + """ + settings_cls = type(self) + values = {} + for name in names: + try: + from_environment = settings_cls.model_validate( + {**current_values, name: raw_values[name]} + ) + except ValidationError: + continue + values[name] = getattr(from_environment, name) + return values + + def save(self) -> None: + """ + Save the current settings to the configuration file of the plugin. + + Only values which differ from the field defaults are persisted. Values + which come from the environment are not written to disk, so credentials + passed as environment variables stay out of the configuration file. + MVT's own configuration file is never modified. + """ + config_folder = plugin_config_folder() + if not os.path.isdir(config_folder): + os.makedirs(config_folder, mode=0o700, exist_ok=True) + + values = json.loads(self.model_dump_json(exclude_defaults=True)) + for name, environment_value in self._environment_values().items(): + if name in values and getattr(self, name, None) == environment_value: + del values[name] + + # Settings files can hold credentials, so write them through a private + # temporary file and move it in place. The file is then never partially + # written and never briefly readable by other users. + config_path = plugin_config_path(self.plugin_name) + descriptor, temporary_path = tempfile.mkstemp( + dir=config_folder, prefix=f".{self.plugin_name}-", suffix=".yaml" + ) + try: + with os.fdopen(descriptor, "w") as config_file: + config_file.write(yaml.dump(values, default_flow_style=False)) + os.replace(temporary_path, config_path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(temporary_path) + raise diff --git a/src/mvt/common/updates.py b/src/mvt/common/updates.py index 001a5c2..71e97cc 100644 --- a/src/mvt/common/updates.py +++ b/src/mvt/common/updates.py @@ -3,8 +3,13 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import importlib.metadata +import json import logging import os +import re +import shlex +import subprocess from datetime import datetime from typing import Optional, Tuple @@ -12,14 +17,35 @@ import requests import yaml from packaging import version +from .cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) from .config import settings from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER +from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url from .version import MVT_VERSION log = logging.getLogger(__name__) # In hours. INDICATORS_CHECK_FREQUENCY = 12 +PLUGINS_CHECK_FREQUENCY = 12 + +# The entry-point groups a package can use to extend MVT. +PLUGIN_ENTRY_POINT_GROUPS = ( + MODULES_ENTRY_POINT_GROUP, + IOS_CLI_PLUGIN_GROUP, + ANDROID_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +SHORT_COMMIT_LENGTH = 8 +# The keys every cached finding has to carry to be printed. +FINDING_KEYS = ("name", "installed", "latest", "upgrade_command") +# Options which stop ssh from waiting for an answer nobody is there to give. +_SSH_BATCH_OPTIONS = ("-o", "BatchMode=yes", "-o", "ConnectTimeout=10") +_COMMIT_PATTERN = re.compile(r"\A[0-9a-f]{7,40}\Z") class MVTUpdates: @@ -38,6 +64,395 @@ class MVTUpdates: return "" +def installed_plugin_distributions() -> list[importlib.metadata.Distribution]: + """Return the installed distributions which extend MVT. + + A plugin package is any distribution registering at least one entry point + in the module or CLI command groups. Distributions are returned once each, + sorted by name. MVT itself is not a plugin and is never returned. + """ + distributions: dict[str, importlib.metadata.Distribution] = {} + + for group in PLUGIN_ENTRY_POINT_GROUPS: + try: + entry_points = importlib.metadata.entry_points(group=group) + except Exception as exc: + log.warning( + "Unable to discover installed plugin packages in entry-point " + "group %s: %s", + group, + exc, + ) + continue + + for entry_point in entry_points: + # Manually constructed entry points have no associated distribution. + dist = getattr(entry_point, "dist", None) + if dist is None: + continue + try: + name = dist.name + except Exception: + continue + if not name or name == "mvt": + continue + distributions.setdefault(name, dist) + + return [distributions[name] for name in sorted(distributions)] + + +def _is_usable_finding(finding: object) -> bool: + """Check that a cached finding carries everything needed to print it.""" + if not isinstance(finding, dict): + return False + + return all( + isinstance(finding.get(key), str) and finding.get(key) for key in FINDING_KEYS + ) + + +def _installed_revision( + dist: importlib.metadata.Distribution, origin: object +) -> Optional[str]: + """Return what a plugin's installed revision is right now. + + This is the value a finding recorded as installed when it was made, so a + finding can be compared against the current state of the installation + without looking anything up remotely. + """ + try: + if origin != "git": + return dist.version + + vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info") + if not isinstance(vcs_info, dict): + return None + + commit = vcs_info.get("commit_id") or "" + return commit[:SHORT_COMMIT_LENGTH] or None + except Exception as e: + log.debug("Failed to read the installed revision of a plugin: %s", e) + return None + + +def _batch_mode_ssh_command(ssh_command: str) -> str: + """Return an ssh command line which cannot stop to ask a question. + + ssh keeps the first value it is given for a keyword, so the batch mode + options are inserted right after the ssh program, ahead of whatever the + analyst configured. Their remaining options, such as which key to use, + still apply. + """ + try: + arguments = shlex.split(ssh_command.strip()) + except ValueError: + arguments = [] + + if not arguments: + arguments = ["ssh"] + + return shlex.join([arguments[0], *_SSH_BATCH_OPTIONS, *arguments[1:]]) + + +def _revision_pins_commit(revision: str, commit: str) -> bool: + """Check whether a requested revision pins the installed commit.""" + candidate = revision.lower() + if not _COMMIT_PATTERN.match(candidate): + return False + + return commit.lower().startswith(candidate) + + +class PluginUpdates: + """Check for updates to the installed MVT plugin packages. + + MVT never installs or upgrades a plugin package itself. It only reports + the command which upgrades a plugin, leaving the analyst to decide when to + run it. + """ + + @property + def latest_check_path(self) -> str: + return os.path.join(MVT_DATA_FOLDER, "latest_plugins_check") + + @property + def findings_path(self) -> str: + return os.path.join(MVT_DATA_FOLDER, "plugin_updates.json") + + def _create_data_folder(self) -> None: + if not os.path.exists(MVT_DATA_FOLDER): + os.makedirs(MVT_DATA_FOLDER) + + def get_latest_check(self) -> int: + if not os.path.exists(self.latest_check_path): + return 0 + + # A corrupt or truncated timestamp only means the next check happens + # sooner. It must never stop MVT from running. + try: + with open(self.latest_check_path, "r", encoding="utf-8") as handle: + data = handle.read().strip() + if data: + return int(data) + except (OSError, ValueError) as e: + log.debug("Failed to read the time of the latest plugin check: %s", e) + + return 0 + + def set_latest_check(self) -> None: + self._create_data_folder() + timestamp = int(datetime.now().timestamp()) + with open(self.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(timestamp)) + + def get_findings(self) -> list[dict]: + """ + Return the findings of the latest check, without checking again. + Returns an empty list if no check was ever performed. + """ + if not os.path.exists(self.findings_path): + return [] + + try: + with open(self.findings_path, "r", encoding="utf-8") as handle: + findings = json.load(handle) + except Exception as e: + log.debug("Failed to read the cached plugin updates: %s", e) + return [] + + if not isinstance(findings, list): + return [] + + # Anything which does not look like a finding is dropped rather than + # trusted: the cache is only a convenience. + return [finding for finding in findings if _is_usable_finding(finding)] + + def current_findings( + self, distributions: Optional[list[importlib.metadata.Distribution]] = None + ) -> list[dict]: + """ + Return the cached findings which still apply to what is installed. + Findings about a plugin which was upgraded or removed since the latest + check are dropped, so an update is never reported twice. + """ + if distributions is None: + distributions = installed_plugin_distributions() + + installed = {} + for dist in distributions: + try: + installed[dist.name] = dist + except Exception: + continue + + current = [] + for finding in self.get_findings(): + plugin = installed.get(finding["name"]) + if plugin is None: + continue + if ( + _installed_revision(plugin, finding.get("origin")) + != finding["installed"] + ): + continue + current.append(finding) + + return current + + def set_findings(self, findings: list[dict]) -> None: + self._create_data_folder() + with open(self.findings_path, "w", encoding="utf-8") as handle: + json.dump(findings, handle) + + def should_check(self) -> Tuple[bool, int]: + """ + Compare time of the latest plugins check with current time. + Returns bool and number of hours since the last check. + """ + now = datetime.now() + latest_check_ts = self.get_latest_check() + latest_check_dt = datetime.fromtimestamp(latest_check_ts) + + diff = now - latest_check_dt + diff_hours = divmod(diff.total_seconds(), 3600)[0] + + if diff_hours >= PLUGINS_CHECK_FREQUENCY: + return True, 0 + + return False, int(PLUGINS_CHECK_FREQUENCY - diff_hours) + + def _check_index_plugin(self, name: str, installed: str) -> Optional[dict]: + """Check a plugin installed from a package index for a newer release.""" + url = f"https://pypi.org/pypi/{name}/json" + try: + res = requests.get(url, timeout=settings.NETWORK_TIMEOUT) + except requests.exceptions.RequestException as e: + log.debug("Failed to check for updates to plugin %s: %s", name, e) + return None + + # Plugins which were never published to a public index are expected, + # and there is nothing to compare their version against. + if res.status_code == 404: + return None + + if res.status_code != 200: + log.debug( + "Failed to check for updates to plugin %s (error %d)", + name, + res.status_code, + ) + return None + + try: + latest = res.json().get("info", {}).get("version", "") + if not latest or version.parse(latest) <= version.parse(installed): + return None + except Exception as e: + log.debug("Failed to compare the versions of plugin %s: %s", name, e) + return None + + return { + "name": name, + "installed": installed, + "latest": latest, + "origin": "pypi", + # The name comes from package metadata, so the command MVT + # suggests is quoted rather than assumed to be shell-safe. + "upgrade_command": f"pip install -U {shlex.quote(name)}", + } + + def _git_ls_remote(self, url: str, revision: str) -> list[Tuple[str, str]]: + """Return the remote references matching a revision, if git allows it.""" + # Neither value is trusted: they are read from the metadata of an + # installed package and must not turn into git options. + if url.startswith("-") or revision.startswith("-"): + log.debug("Skipping the update check for the invalid repository %s", url) + return [] + + environment = dict(os.environ) + # Never prompt the analyst for repository credentials. git handles its + # own prompts, while ssh reads the terminal directly and only batch + # mode makes it fail instead of asking. + environment["GIT_TERMINAL_PROMPT"] = "0" + environment["GIT_SSH_COMMAND"] = _batch_mode_ssh_command( + environment.get("GIT_SSH_COMMAND", "") + ) + + try: + process = subprocess.run( + ["git", "ls-remote", url, revision], + capture_output=True, + stdin=subprocess.DEVNULL, + text=True, + env=environment, + timeout=settings.NETWORK_TIMEOUT, + check=False, + ) + except FileNotFoundError: + log.debug("Could not find git, skipping the update check for %s", url) + return [] + except (subprocess.SubprocessError, OSError) as e: + log.debug("Failed to query the repository %s: %s", url, e) + return [] + + if process.returncode != 0: + log.debug( + "Failed to query the repository %s (error %d): %s", + url, + process.returncode, + (process.stderr or "").strip(), + ) + return [] + + references = [] + for line in (process.stdout or "").splitlines(): + commit, _, reference = line.partition("\t") + if commit.strip() and reference.strip(): + references.append((commit.strip(), reference.strip())) + + return references + + def _check_repository_plugin( + self, name: str, direct_url: dict, vcs_info: dict + ) -> Optional[dict]: + """Check a plugin installed from a repository for a newer commit.""" + url = direct_url.get("url") or "" + installed = vcs_info.get("commit_id") or "" + revision = vcs_info.get("requested_revision") or "" + if not url or not installed: + return None + + # A plugin installed from a commit is pinned and never goes out of + # date, no matter what the branch it came from does next. + if revision and _revision_pins_commit(revision, installed): + return None + + latest = "" + wanted_reference = f"refs/heads/{revision}" if revision else "HEAD" + for commit, reference in self._git_ls_remote(url, revision or "HEAD"): + # Tags are pinned installs too. + if reference.startswith("refs/tags/"): + return None + if reference == wanted_reference: + latest = commit + + if not latest or latest == installed: + return None + + requirement = f"{name} @ git+{url}" + if revision: + requirement += f"@{revision}" + + return { + "name": name, + "installed": installed[:SHORT_COMMIT_LENGTH], + "latest": latest[:SHORT_COMMIT_LENGTH], + "origin": "git", + # A repository URL and a branch name can both hold characters a + # shell would act on, so the requirement is quoted for the shell + # the analyst is going to paste the command into. + "upgrade_command": f"pip install -U {shlex.quote(requirement)}", + } + + def _check_distribution( + self, dist: importlib.metadata.Distribution + ) -> Optional[dict]: + try: + name = dist.name + installed = dist.version + except Exception as e: + log.debug("Failed to read the metadata of an installed plugin: %s", e) + return None + + direct_url = distribution_direct_url(dist) + if direct_url is None: + return self._check_index_plugin(name, installed) + + vcs_info = direct_url.get("vcs_info") + if isinstance(vcs_info, dict): + return self._check_repository_plugin(name, direct_url, vcs_info) + + # Plugins installed from a local folder, including editable installs, + # are maintained by the analyst and have nothing to check against. + return None + + def check(self) -> list[dict]: + """ + Check every installed plugin package for an available update. + Returns one entry per plugin which can be upgraded. + """ + findings = [] + for dist in installed_plugin_distributions(): + finding = self._check_distribution(dist) + if finding: + findings.append(finding) + + self.set_findings(findings) + self.set_latest_check() + + return findings + + class IndicatorsUpdates: def __init__(self) -> None: self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}" @@ -180,9 +595,7 @@ class IndicatorsUpdates: def _get_remote_file_latest_commit( self, owner: str, repo: str, branch: str, path: str ) -> int: - file_commit_url = ( - f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" - ) + file_commit_url = f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" try: res = requests.get(file_commit_url, timeout=5) except requests.exceptions.RequestException as e: diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index 30de159..2a3e39a 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -14,7 +14,6 @@ from dataclasses import asdict, is_dataclass from typing import Any, Iterator, Union from .log import MVTLogHandler -from mvt.common.config import settings class CustomJSONEncoder(json.JSONEncoder): @@ -239,6 +238,13 @@ def init_logging(verbose: bool = False): """ log = logging.getLogger("mvt") log.setLevel(logging.DEBUG) + + # Importing an MVT CLI module calls init_logging() at import time, and + # loaded module packages may import one indirectly. Keep this idempotent + # so console log lines are not duplicated by a second handler. + if any(isinstance(handler, MVTLogHandler) for handler in log.handlers): + return + consoleHandler = MVTLogHandler() consoleHandler.setFormatter(logging.Formatter("%(message)s")) if verbose: @@ -249,16 +255,26 @@ def init_logging(verbose: bool = False): def set_verbose_logging(verbose: bool = False): + """Raise or lower the verbosity of MVT's console output. + + Only MVT's own console handler is adjusted, wherever it sits in the list. + The file handler a command attaches to its output folder keeps recording + everything, so the command.log of a run does not depend on how the run was + invoked, and a handler attached to the "mvt" logger by anything else is + left alone. + """ log = logging.getLogger("mvt") - handler = log.handlers[0] - if verbose: - handler.setLevel(logging.DEBUG) - else: - handler.setLevel(logging.INFO) + for handler in log.handlers: + if isinstance(handler, MVTLogHandler): + handler.setLevel(logging.DEBUG if verbose else logging.INFO) def exec_or_profile(module, globals, locals): """Hook for profiling MVT modules""" + # Imported here so that the CLI modules, which import this one at start-up, + # do not load the settings (and pydantic) before a command runs. + from .config import settings + if settings.PROFILE: cProfile.runctx(module, globals, locals) else: diff --git a/src/mvt/ios/__init__.py b/src/mvt/ios/__init__.py index 2c05f56..6616bcb 100644 --- a/src/mvt/ios/__init__.py +++ b/src/mvt/ios/__init__.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from .cli import cli +from .cli import cli, main diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index c338fa5..edf7724 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -14,16 +14,7 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.completion import ( - SUPPORTED_SHELLS, - completion_instructions, - generate_completion_script, - install_completion_script, -) -from mvt.common.logo import logo from mvt.common.options import MutuallyExclusiveOption -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import ( generate_hashes_from_path, init_logging, @@ -44,6 +35,7 @@ from mvt.common.help import ( HELP_MSG_LOAD_MODULE, HELP_MSG_MODULE, HELP_MSG_VERBOSE, + HELP_MSG_VERBOSE_COMMAND, HELP_MSG_CHECK_FS, HELP_MSG_CHECK_IOCS, HELP_MSG_STIX2, @@ -51,17 +43,13 @@ from mvt.common.help import ( HELP_MSG_CHECK_SYSDIAGNOSE, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, - HELP_MSG_COMPLETION, ) -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from mvt.common.password import prompt_password -from .cmd_check_backup import CmdIOSCheckBackup -from .cmd_check_fs import CmdIOSCheckFS -from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose -from .decrypt import DecryptBackup -from .modules.backup import BACKUP_MODULES -from .modules.fs import FS_MODULES -from .modules.mixed import MIXED_MODULES + +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-ios, including by shell completion on every +# keystroke, so importing it must do no more than build the command tree: the +# forensic modules, the backup decryption and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -81,7 +69,14 @@ def _get_disable_flags(ctx): ) +def _get_verbose(ctx): + """Return whether --verbose was passed to the CLI itself.""" + return bool(ctx.obj and ctx.obj.get("verbose", False)) + + def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -101,60 +96,31 @@ def _load_custom_modules(load_module): is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - if ctx.invoked_subcommand != "completion": - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) + + from mvt.common.logo import logo + + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== # Command: version # ============================================================================== -@cli.command("version", help=HELP_MSG_VERSION) +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) def version(): return -# ============================================================================== -# Command: completion -# ============================================================================== -@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) -@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) -@click.option( - "--install", - is_flag=True, - help="Write completion files and update shell configuration.", -) -@click.pass_context -def completion(ctx, shell, install): - program_name = "mvt-ios" - - if shell is None: - if install: - raise click.UsageError("A shell is required when using --install.") - click.echo(completion_instructions(program_name)) - return - - root_cli = ctx.find_root().command - - if install: - script_path = install_completion_script(root_cli, program_name, shell) - click.echo(f"Installed {shell} completion to {script_path}") - if shell in ("bash", "zsh"): - click.echo(f"Updated ~/.{shell}rc") - else: - click.echo("Fish loads completion files automatically.") - return - - click.echo(generate_completion_script(root_cli, program_name, shell)) - - # ============================================================================== # Command: decrypt-backup # ============================================================================== @@ -181,6 +147,8 @@ def completion(ctx, shell, install): @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path, destination) if key_file: @@ -244,6 +212,8 @@ def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path): ) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) def extract_key(password, key_file, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path) if password: @@ -296,7 +266,7 @@ def extract_key(password, key_file, backup_path): help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( @@ -311,7 +281,9 @@ def check_backup( verbose, backup_path, ): - set_verbose_logging(verbose) + from .cmd_check_backup import CmdIOSCheckBackup + + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -365,7 +337,7 @@ def check_backup( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("DUMP_PATH", type=click.Path(exists=True)) @click.pass_context def check_fs( @@ -380,7 +352,9 @@ def check_fs( verbose, dump_path, ): - set_verbose_logging(verbose) + from .cmd_check_fs import CmdIOSCheckFS + + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -434,7 +408,7 @@ def check_fs( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("SYSDIAGNOSE_PATH", type=click.Path(exists=True)) @click.pass_context def check_sysdiagnose( @@ -448,7 +422,9 @@ def check_sysdiagnose( verbose, sysdiagnose_path, ): - set_verbose_logging(verbose) + from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckSysdiagnose( target_path=sysdiagnose_path, @@ -502,6 +478,10 @@ def check_sysdiagnose( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import IOS_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -512,7 +492,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): custom_modules=custom_modules, platform="ios", ) - cmd.modules = BACKUP_MODULES + FS_MODULES + MIXED_MODULES + cmd.modules = IOS_CHECK_IOCS_MODULES if list_modules: cmd.list_modules() @@ -528,12 +508,25 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_iocs(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() -register_cli_plugins( - cli, - entry_point_group=IOS_CLI_PLUGIN_GROUP, - environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV, -) +# ============================================================================== +# Entry point of the mvt-ios console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt-ios CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/ios/command_modules.py b/src/mvt/ios/command_modules.py new file mode 100644 index 0000000..26fa1d6 --- /dev/null +++ b/src/mvt/ios/command_modules.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Module lists an mvt-ios command composes from more than one family. + +Commands whose modules are one family read that family directly. check-iocs +re-checks stored results, so it has to know every module that could have +written one, and both the CLI and any other code needing that answer share +the list from here rather than each concatenating their own. +""" + +from mvt.common.module import MVTModule + +from .modules.backup import BACKUP_MODULES +from .modules.fs import FS_MODULES +from .modules.mixed import MIXED_MODULES + +IOS_CHECK_IOCS_MODULES: list[type[MVTModule]] = ( + BACKUP_MODULES + FS_MODULES + MIXED_MODULES +) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index 16fbba5..9b4613b 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1220,6 +1220,10 @@ "version": "18.7.10", "build": "22H373" }, + { + "version": "18.7.10", + "build": "22H374" + }, { "version": "26", "build": "23A341" @@ -1263,5 +1267,9 @@ { "version": "26.6.1", "build": "23G82" + }, + { + "version": "26.6.1", + "build": "23G83" } ] \ No newline at end of file diff --git a/src/mvt/ios/modules/mixed/__init__.py b/src/mvt/ios/modules/mixed/__init__.py index 0e50c26..a846bf2 100644 --- a/src/mvt/ios/modules/mixed/__init__.py +++ b/src/mvt/ios/modules/mixed/__init__.py @@ -26,6 +26,7 @@ from .tcc import TCC from .webkit_resource_load_statistics import WebkitResourceLoadStatistics from .webkit_session_resource_log import WebkitSessionResourceLog from .whatsapp import Whatsapp +from .whatsapp_contacts import WhatsappContacts MIXED_MODULES = [ Calls, @@ -47,6 +48,7 @@ MIXED_MODULES = [ WebkitResourceLoadStatistics, WebkitSessionResourceLog, Whatsapp, + WhatsappContacts, Shortcuts, Applications, Calendar, diff --git a/src/mvt/ios/modules/mixed/interactionc.py b/src/mvt/ios/modules/mixed/interactionc.py index 81a67e2..4d21382 100644 --- a/src/mvt/ios/modules/mixed/interactionc.py +++ b/src/mvt/ios/modules/mixed/interactionc.py @@ -3,9 +3,11 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import datetime import logging +import re import sqlite3 -from typing import Optional +from typing import Optional, Tuple from mvt.common.module_types import ( ModuleAtomicResult, @@ -15,6 +17,7 @@ from mvt.common.module_types import ( from mvt.common.utils import convert_mactime_to_iso from ..base import IOSExtraction +from .whatsapp_contacts import WhatsappContacts INTERACTIONC_BACKUP_IDS = [ "1f5a521220a3ad80ebfdc196978df8e7a2e49dee", @@ -22,6 +25,49 @@ INTERACTIONC_BACKUP_IDS = [ INTERACTIONC_ROOT_PATHS = [ "private/var/mobile/Library/CoreDuet/People/interactionC.db", ] + +# The interaction record's creation date normally trails its start date by +# milliseconds: emitting it as a timeline event only duplicates the start +# date event. A large divergence, however, indicates the record was +# backfilled (sync, restore, or tampering) and is worth surfacing. +CREATION_DATE_DIVERGENCE_THRESHOLD = 3600.0 + +# Per-contact aggregate dates from ZCONTACTS are repeated on every +# interaction row of the same contact. They are serialized with a +# contact-centric data string so that timeline de-duplication collapses +# them into one event per contact. +CONTACT_EVENT_TEMPLATES = { + "contacts_creation_date": "Contact {party} first recorded in interactionC", + "first_incoming_sender_date": "First incoming interaction from {party}", + "last_incoming_sender_date": "Last incoming interaction from {party}", + "first_incoming_recipient_date": ( + "First incoming interaction where {party} was a recipient" + ), + "last_incoming_recipient_date": ( + "Last incoming interaction where {party} was a recipient" + ), + "first_outgoing_recipient_date": ( + "First outgoing interaction to {party}" + ), + "last_outgoing_recipient_date": ( + "Last outgoing interaction to {party}" + ), +} + + +def _parse_iso(timestamp) -> Optional[datetime.datetime]: + try: + return datetime.datetime.strptime( + timestamp, "%Y-%m-%d %H:%M:%S.%f" + ) + except (TypeError, ValueError): + return None + + +def _describe_delta(seconds: float) -> str: + if seconds >= 86400: + return f"{seconds / 86400:.0f} days" + return f"{seconds / 3600:.0f} hours" # Taken from APOLLO # https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt QUERIES = [ @@ -34,7 +80,7 @@ QUERIES = [ CASE ZINTERACTIONS.ZDIRECTION WHEN '0' THEN 'INCOMING' WHEN '1' THEN 'OUTGOING' - END 'DIRECTION' AS "direction", + END AS "direction", ZCONTACTS.ZDISPLAYNAME AS "sender_display_name", ZCONTACTS.ZIDENTIFIER AS "sender_identifier", ZCONTACTS.ZPERSONID AS "sender_personid", @@ -89,7 +135,7 @@ QUERIES = [ CASE ZINTERACTIONS.ZDIRECTION WHEN '0' THEN 'INCOMING' WHEN '1' THEN 'OUTGOING' - END 'DIRECTION' AS "direction", + END AS "direction", ZCONTACTS.ZDISPLAYNAME AS "sender_display_name", ZCONTACTS.ZIDENTIFIER AS "sender_identifier", ZCONTACTS.ZPERSONID AS "sender_personid", @@ -117,7 +163,7 @@ QUERIES = [ CASE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE WHEN '0' THEN '0' ELSE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE - END 'LAST INCOMING RECIPIENT DATE' AS "last_incoming_recipient_date", + END AS "last_incoming_recipient_date", ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE AS "last_outgoing_recipient_date", ZCONTACTS.ZCUSTOMIDENTIFIER AS "custom_id", ZINTERACTIONS.ZCONTENTURL AS "interaction_content_url", @@ -218,9 +264,16 @@ QUERIES = [ ] +WHATSAPP_BUNDLE_ID = "net.whatsapp.WhatsApp" + + class InteractionC(IOSExtraction): """This module extracts data from InteractionC db.""" + # WhatsApp identifies chat peers by LID in interactionC.db, which only the + # WhatsApp contacts database can map back to a phone number and name. + dependencies = [WhatsappContacts] + def __init__( self, file_path: Optional[str] = None, @@ -252,10 +305,51 @@ class InteractionC(IOSExtraction): "last_outgoing_recipient_date", ] + @staticmethod + def _describe_party(record: ModuleAtomicResult, prefix: str) -> Optional[str]: + name = record.get(f"{prefix}_display_name") or record.get( + f"{prefix}_resolved_name" + ) + identifier = record.get(f"{prefix}_resolved_phone_number") or record.get( + f"{prefix}_identifier" + ) + if name and identifier: + # A display name that is just a formatted copy of the phone + # number adds no information. + name_digits = re.sub(r"\D", "", name) + if name_digits and name_digits == re.sub(r"\D", "", identifier): + return identifier + return f"{name} ({identifier})" + return name or identifier or None + def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + sender = self._describe_party(record, "sender") + # The chat peer from the domain identifier stands in when the + # recipient was not recorded (or the recipient join is unavailable). + recipient = self._describe_party(record, "recipient") or self._describe_party( + record, "domain" + ) + direction = record.get("direction") + if not sender and direction == "OUTGOING": + sender = "local user" + if not recipient and direction == "INCOMING": + recipient = "local user" + + header = f"[{record['bundle_id']}]" + if record.get("account"): + header += f" {record['account']}" + if direction: + header += f" {direction}" + + data = f"{header} from {sender or 'unknown'} to {recipient or 'unknown'}" + if record.get("group_name"): + data += f" (group: {record['group_name']})" + if record.get("content"): + data += f": {record['content']}" + records = [] processed = [] - for timestamp in self.timestamps: + for timestamp in ("start_date", "end_date"): # Check if the record has the current timestamp. if timestamp not in record or not record[timestamp]: continue @@ -269,16 +363,142 @@ class InteractionC(IOSExtraction): "timestamp": record[timestamp], "module": self.__class__.__name__, "event": timestamp, - "data": f"[{record['bundle_id']}] {record['account']} - " - f"from {record['sender_display_name']} ({record['sender_identifier']}) " - f"to {record.get('recipient_display_name', '')} ({record.get('recipient_identifier', '')}):" - f" {record.get('content', '')}", + "data": data, } ) processed.append(record[timestamp]) + creation_event = self._serialize_creation_date(record, data) + if creation_event: + records.append(creation_event) + + # Contact-level aggregates describe the sender's contact record. + party = self._describe_party(record, "sender") + if party: + for field, template in CONTACT_EVENT_TEMPLATES.items(): + if not record.get(field): + continue + records.append( + { + "timestamp": record[field], + "module": self.__class__.__name__, + "event": field, + "data": template.format(party=party), + } + ) + return records + def _serialize_creation_date( + self, record: ModuleAtomicResult, data: str + ) -> Optional[dict]: + """Serialize the interaction record's creation date only when it + diverges from the start date enough to indicate the record was + backfilled.""" + creation = record.get("interactions_creation_date") + if not creation: + return None + + event = { + "timestamp": creation, + "module": self.__class__.__name__, + "event": "interactions_creation_date", + "data": data, + } + + start = _parse_iso(record.get("start_date")) + creation_parsed = _parse_iso(creation) + if not start or not creation_parsed: + # Without a start date the creation date is the only anchor. + return event + + delta = (creation_parsed - start).total_seconds() + if abs(delta) < CREATION_DATE_DIVERGENCE_THRESHOLD: + return None + + direction = "after" if delta > 0 else "before" + event["data"] = ( + f"Interaction record created {_describe_delta(abs(delta))} " + f"{direction} the event: {data}" + ) + return event + + def _whatsapp_contact_maps(self) -> Tuple[dict, dict]: + """Build LID and phone-digit lookup maps from the WhatsappContacts + module results, when available.""" + by_lid: dict = {} + by_phone: dict = {} + contacts_module = self.dependency_modules.get(WhatsappContacts) + if not contacts_module: + return by_lid, by_phone + + for contact in contacts_module.results: + name = contact.get("full_name") or contact.get("given_name") + phone = contact.get("phone_number") + entry = (phone, name) + if contact.get("lid"): + by_lid[contact["lid"]] = entry + if phone: + by_phone[re.sub(r"\D", "", phone)] = entry + whatsapp_id = contact.get("whatsapp_id") + if whatsapp_id and "@" in whatsapp_id: + by_phone.setdefault(whatsapp_id.split("@")[0], entry) + + return by_lid, by_phone + + @staticmethod + def _resolve_whatsapp_identifier( + value, by_lid: dict, by_phone: dict + ) -> Tuple[Optional[str], Optional[str]]: + """Resolve a WhatsApp identifier (LID, JID or phone number) to a + (phone_number, contact_name) tuple.""" + if not value: + return None, None + + value = str(value) + if value.endswith("@lid"): + return by_lid.get(value, (None, None)) + if value.endswith("@g.us"): + return None, None + if value.endswith("@s.whatsapp.net"): + digits = value.split("@")[0] + phone, name = by_phone.get(digits, (None, None)) + return phone or f"+{digits}", name + if value.startswith("+"): + _, name = by_phone.get(re.sub(r"\D", "", value), (None, None)) + return None, name + + return None, None + + def _postprocess_results(self) -> None: + by_lid, by_phone = self._whatsapp_contact_maps() + + for entry in self.results: + # The fallback queries return ZDIRECTION raw instead of labelled. + if entry.get("direction") in (0, "0"): + entry["direction"] = "INCOMING" + elif entry.get("direction") in (1, "1"): + entry["direction"] = "OUTGOING" + + if entry.get("bundle_id") != WHATSAPP_BUNDLE_ID: + continue + + candidates = { + "sender": [entry.get("sender_identifier"), entry.get("custom_id")], + "recipient": [entry.get("recipient_identifier")], + "domain": [entry.get("domain_identifier")], + } + for prefix, values in candidates.items(): + phone = name = None + for value in values: + phone, name = self._resolve_whatsapp_identifier( + value, by_lid, by_phone + ) + if phone or name: + break + entry[f"{prefix}_resolved_phone_number"] = phone + entry[f"{prefix}_resolved_name"] = name + def run(self) -> None: self._find_ios_database( backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS @@ -325,4 +545,6 @@ class InteractionC(IOSExtraction): cur.close() conn.close() + self._postprocess_results() + self.log.info("Extracted a total of %d InteractionC events", len(self.results)) diff --git a/src/mvt/ios/modules/mixed/whatsapp.py b/src/mvt/ios/modules/mixed/whatsapp.py index 449f08f..5cf81cf 100644 --- a/src/mvt/ios/modules/mixed/whatsapp.py +++ b/src/mvt/ios/modules/mixed/whatsapp.py @@ -4,7 +4,9 @@ # https://license.mvt.re/1.1/ import logging -from typing import Optional +import os +import sqlite3 +from typing import Dict, Optional from mvt.common.module_types import ( ModuleAtomicResult, @@ -22,9 +24,71 @@ WHATSAPP_ROOT_PATHS = [ "private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite", ] +WHATSAPP_LID_BACKUP_IDS = [ + # SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-LID.sqlite" + "e794f6ffcc3c222535f47684a63d5178da3c4500", +] +WHATSAPP_LID_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/LID.sqlite", +] + +# WhatsApp records the mapping between a contact's LID and phone number +# identifiers in the ZWAPHONENUMBERLIDPAIR table. Depending on the WhatsApp +# version this lives in a dedicated LID.sqlite database or in +# ChatStorage.sqlite itself. +LID_PAIRS_QUERY = """ + SELECT + ZLID AS "lid", + ZPHONENUMBER AS "phone_number", + ZTIMESTAMP AS "pair_timestamp" + FROM ZWAPHONENUMBERLIDPAIR; +""" + +CHAT_SESSIONS_QUERY = """ + SELECT + ZWACHATSESSION.Z_PK AS "session_pk", + ZWACHATSESSION.ZCONTACTJID AS "contact_jid", + ZWACHATSESSION.ZPARTNERNAME AS "partner_name", + ZWACHATSESSION.ZSESSIONTYPE AS "session_type", + ZWACHATSESSION.ZARCHIVED AS "archived", + ZWACHATSESSION.ZREMOVED AS "removed", + ZWACHATSESSION.ZMESSAGECOUNTER AS "message_counter", + ZWACHATSESSION.ZLASTMESSAGEDATE AS "last_message_date", + ZWAGROUPINFO.ZCREATIONDATE AS "group_creation_date", + MIN(ZWAMESSAGE.ZMESSAGEDATE) AS "first_stored_message_date", + MAX(ZWAMESSAGE.ZMESSAGEDATE) AS "last_stored_message_date", + COUNT(ZWAMESSAGE.Z_PK) AS "stored_message_count" + FROM ZWACHATSESSION + LEFT JOIN ZWAGROUPINFO + ON ZWACHATSESSION.ZGROUPINFO = ZWAGROUPINFO.Z_PK + LEFT JOIN ZWAMESSAGE + ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK + GROUP BY ZWACHATSESSION.Z_PK; +""" + +CHAT_SESSION_DATE_FIELDS = [ + "last_message_date", + "group_creation_date", + "first_stored_message_date", + "last_stored_message_date", +] + + +def _describe_chat(record: ModuleAtomicResult) -> str: + jid = record.get("contact_jid") or "unknown" + name = record.get("partner_name") + identifier = record.get("partner_resolved_phone_number") or jid + label = f"'{name}' ({identifier})" if name else identifier + is_group = jid.endswith("@g.us") or record.get("group_creation_date") + if is_group: + return f"WhatsApp group chat {label}" + return f"WhatsApp chat with {label}" + class Whatsapp(IOSExtraction): - """This module extracts all WhatsApp messages containing links.""" + """This module extracts all WhatsApp messages containing links, as well + as per-chat records with the first and last interaction dates of each + conversation.""" def __init__( self, @@ -45,6 +109,21 @@ class Whatsapp(IOSExtraction): ) def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + if record.get("record_type") == "chat_session": + return self._serialize_chat_session(record) + if record.get("record_type") == "lid_phone_number_pair": + if not record.get("pair_timestamp"): + return [] + return { + "timestamp": record["pair_timestamp"], + "module": self.__class__.__name__, + "event": "lid_pair_recorded", + "data": ( + f"WhatsApp associated LID {record.get('lid')} with " + f"phone number {record.get('phone_number')}" + ), + } + text = record.get("ZTEXT", "").replace("\n", "\\n") links_text = "" if record.get("links"): @@ -57,6 +136,49 @@ class Whatsapp(IOSExtraction): "data": f"'{text}' from {record.get('ZFROMJID', 'Unknown')}{links_text}", } + def _serialize_chat_session( + self, record: ModuleAtomicResult + ) -> ModuleSerializedResult: + records = [] + chat = _describe_chat(record) + + if record.get("group_creation_date"): + records.append( + { + "timestamp": record["group_creation_date"], + "module": self.__class__.__name__, + "event": "group_created", + "data": f"{chat} was created", + } + ) + + if record.get("first_stored_message_date"): + records.append( + { + "timestamp": record["first_stored_message_date"], + "module": self.__class__.__name__, + "event": "chat_first_message", + "data": f"First stored message in {chat}", + } + ) + + # The chat session's own last-message date is authoritative: it can + # postdate the newest stored message if that message was deleted. + last_message_date = record.get("last_message_date") or record.get( + "last_stored_message_date" + ) + if last_message_date: + records.append( + { + "timestamp": last_message_date, + "module": self.__class__.__name__, + "event": "chat_last_message", + "data": f"Last message in {chat}", + } + ) + + return records + def check_indicators(self) -> None: if not self.indicators: return @@ -149,7 +271,126 @@ class Whatsapp(IOSExtraction): message["links"] = list(set(filtered_links)) self.results.append(message) + total_messages = len(self.results) + lid_map = self._extract_lid_pairs(cur) + total_sessions = self._extract_chat_sessions(cur, lid_map) + cur.close() conn.close() - self.log.info("Extracted a total of %d WhatsApp messages", len(self.results)) + self.log.info( + "Extracted a total of %d WhatsApp messages, %d chat sessions " + "and %d LID-phone number pairs", + total_messages, + total_sessions, + len(lid_map), + ) + + def _find_lid_db_path(self) -> Optional[str]: + for backup_id in WHATSAPP_LID_BACKUP_IDS: + file_path = self._get_backup_file_from_id(backup_id) + if file_path and os.path.exists(file_path): + return file_path + for found_path in self._get_fs_files_from_patterns( + WHATSAPP_LID_ROOT_PATHS + ): + return found_path + return None + + def _extract_lid_pairs(self, chat_cur: sqlite3.Cursor) -> Dict[str, str]: + """Extract the LID to phone number mapping from the dedicated + LID.sqlite database, falling back to the same table in + ChatStorage.sqlite. Returns a map of LID digits to phone number + digits.""" + rows = [] + lid_db_path = self._find_lid_db_path() + if lid_db_path: + self.log.info( + "Found WhatsApp LID database at path: %s", lid_db_path + ) + lid_conn = self._open_sqlite_db(lid_db_path) + try: + lid_cur = lid_conn.cursor() + lid_cur.execute(LID_PAIRS_QUERY) + rows = lid_cur.fetchall() + lid_cur.close() + except sqlite3.DatabaseError as exc: + self.log.warning( + "Unable to extract WhatsApp LID-phone number pairs: %s", + exc, + ) + finally: + lid_conn.close() + else: + try: + chat_cur.execute(LID_PAIRS_QUERY) + rows = chat_cur.fetchall() + except sqlite3.OperationalError: + self.log.info( + "No WhatsApp LID database found in this backup or " + "filesystem dump: LID chat identifiers cannot be " + "resolved to phone numbers" + ) + + lid_map: Dict[str, str] = {} + for lid, phone_number, pair_timestamp in rows: + record = { + "record_type": "lid_phone_number_pair", + "lid": lid, + "phone_number": phone_number, + "pair_timestamp": ( + convert_mactime_to_iso(pair_timestamp) or None + ) + if pair_timestamp + else None, + } + self.results.append(record) + + if lid and phone_number: + lid_digits = str(lid).split("@")[0] + phone_digits = str(phone_number).split("@")[0].lstrip("+") + lid_map[lid_digits] = phone_digits + + return lid_map + + def _extract_chat_sessions( + self, cur: sqlite3.Cursor, lid_map: Dict[str, str] + ) -> int: + """Extract one record per chat session with the first and last + interaction dates of each conversation.""" + try: + cur.execute(CHAT_SESSIONS_QUERY) + except sqlite3.OperationalError as exc: + self.log.warning( + "Unable to extract WhatsApp chat sessions: %s", exc + ) + return 0 + + names = [description[0] for description in cur.description] + + total_sessions = 0 + for row in cur.fetchall(): + session = dict(zip(names, row)) + session["record_type"] = "chat_session" + + session["partner_resolved_phone_number"] = None + jid = session.get("contact_jid") or "" + if jid.endswith("@lid"): + phone_digits = lid_map.get(jid.split("@")[0]) + if phone_digits: + session["partner_resolved_phone_number"] = ( + f"+{phone_digits}" + ) + + for field in CHAT_SESSION_DATE_FIELDS: + if session.get(field): + session[field] = ( + convert_mactime_to_iso(session[field]) or None + ) + else: + session[field] = None + + self.results.append(session) + total_sessions += 1 + + return total_sessions diff --git a/src/mvt/ios/modules/mixed/whatsapp_contacts.py b/src/mvt/ios/modules/mixed/whatsapp_contacts.py new file mode 100644 index 0000000..309da22 --- /dev/null +++ b/src/mvt/ios/modules/mixed/whatsapp_contacts.py @@ -0,0 +1,308 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +import sqlite3 +from typing import Optional + +from mvt.common.module import DatabaseNotFoundError +from mvt.common.module_types import ( + ModuleAtomicResult, + ModuleResults, + ModuleSerializedResult, +) +from mvt.common.utils import convert_mactime_to_iso + +from ..base import IOSExtraction + +WHATSAPP_CONTACTS_BACKUP_IDS = [ + # SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-ContactsV2.sqlite" + "b8548dc30aa1030df0ce18ef08b882cf7ab5212f", +] +WHATSAPP_CONTACTS_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/ContactsV2.sqlite", +] + +# WhatsApp's standard disappearing-messages timer values, in seconds. +DISAPPEARING_DURATION_LABELS = { + 86400: "24 hours", + 604800: "7 days", + 1209600: "14 days", + 2592000: "30 days", + 7776000: "90 days", +} + +# Output field -> candidate columns in ZWAADDRESSBOOKCONTACT, in order of +# preference. WhatsApp renames columns across versions, so the query is built +# from the columns actually present in the database. +COLUMN_CANDIDATES = { + "whatsapp_id": ["ZWHATSAPPID"], + "lid": ["ZLID"], + "phone_number": ["ZPHONENUMBER"], + "localized_phone_number": ["ZLOCALIZEDPHONENUMBER"], + "full_name": ["ZFULLNAME"], + "given_name": ["ZGIVENNAME"], + "last_name": ["ZLASTNAME"], + "user_name": ["ZUSERNAME"], + "business_name": ["ZBUSINESSNAME"], + "about_text": ["ZABOUTTEXT"], + "about_emoji": ["ZABOUTEMOJI"], + "notes": ["ZNOTES"], + "disappearing_mode_duration": ["ZDISAPPEARINGMODEDURATION"], + "disappearing_mode_timestamp": ["ZDISAPPEARINGMODETIMESTAMP"], + "about_timestamp": ["ZABOUTTIMESTAMP"], + "about_expiration_timestamp": ["ZABOUTEXPIRATIONTIMESTAMP"], + "last_updated": ["ZLASTUPDATED"], + "phone_status": ["ZPHONESTATUS", "ZPHONENUMBERSTATUS"], + "sync_policy": ["ZSYNCPOLICY"], +} + +STRING_FIELDS = [ + "whatsapp_id", + "lid", + "phone_number", + "localized_phone_number", + "full_name", + "given_name", + "last_name", + "user_name", + "business_name", + "about_text", + "about_emoji", + "notes", +] + +DATE_FIELDS = [ + "disappearing_mode_timestamp", + "about_timestamp", + "about_expiration_timestamp", + "last_updated", +] + + +def _decode_string(value) -> Optional[str]: + # CoreData stores string attributes as UTF-8 blobs in some WhatsApp + # versions, so values can arrive as either bytes or str. + if value is None: + return None + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) + + +def _label_duration(duration) -> str: + if not duration: + return "off" + return DISAPPEARING_DURATION_LABELS.get( + int(duration), f"{int(duration)} seconds" + ) + + +def _describe_contact(record: ModuleAtomicResult) -> str: + contact = ( + record.get("whatsapp_id") + or record.get("lid") + or record.get("phone_number") + or "unknown" + ) + full_name = record.get("full_name") + if full_name: + contact = f"{contact} ({full_name})" + return contact + + +class WhatsappContacts(IOSExtraction): + """This module extracts WhatsApp contact records and per-contact + disappearing-messages settings from ContactsV2.sqlite. + + ChatStorage.sqlite does not record the disappearing-messages state of 1:1 + chats: the authoritative timer is stored on each contact record in this + database, alongside the mapping between a contact's LID and phone number + identifiers. + """ + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[ModuleResults] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + records = [] + contact = _describe_contact(record) + + if record.get("disappearing_mode_timestamp"): + records.append( + { + "timestamp": record["disappearing_mode_timestamp"], + "module": self.__class__.__name__, + "event": "disappearing_mode_set", + "data": ( + f"WhatsApp disappearing messages timer set to " + f"'{record.get('disappearing_mode_label')}' " + f"for {contact}" + ), + } + ) + + if record.get("about_timestamp"): + data = f"WhatsApp about text of {contact} changed" + about_text = record.get("about_text") + if about_text: + data += f' to "{about_text}"' + records.append( + { + "timestamp": record["about_timestamp"], + "module": self.__class__.__name__, + "event": "about_changed", + "data": data, + } + ) + + if record.get("about_expiration_timestamp"): + records.append( + { + "timestamp": record["about_expiration_timestamp"], + "module": self.__class__.__name__, + "event": "about_expiration", + "data": ( + f"WhatsApp about text of {contact} scheduled " + f"to expire" + ), + } + ) + + if record.get("last_updated"): + records.append( + { + "timestamp": record["last_updated"], + "module": self.__class__.__name__, + "event": "contact_last_updated", + "data": f"WhatsApp contact record for {contact} updated", + } + ) + + return records + + def run(self) -> None: + try: + self._find_ios_database( + backup_ids=WHATSAPP_CONTACTS_BACKUP_IDS, + root_paths=WHATSAPP_CONTACTS_ROOT_PATHS, + ) + except DatabaseNotFoundError: + self.log.warning( + "Unable to find the WhatsApp ContactsV2.sqlite database in " + "this backup or filesystem dump. WhatsApp disappearing " + "messages settings and contact records cannot be extracted. " + "This database is often missing from incremental backups." + ) + return + + self.log.info( + "Found WhatsApp contacts database at path: %s", self.file_path + ) + + assert self.file_path is not None + conn = self._open_sqlite_db(self.file_path) + cur = conn.cursor() + try: + try: + cur.execute("PRAGMA table_info(ZWAADDRESSBOOKCONTACT)") + available_columns = {row[1] for row in cur.fetchall()} + except sqlite3.DatabaseError as exc: + self.log.error( + "Unable to read the ZWAADDRESSBOOKCONTACT table schema: %s", + exc, + ) + return + + if not available_columns: + self.log.warning( + "The WhatsApp contacts database does not contain a " + "ZWAADDRESSBOOKCONTACT table" + ) + return + + selected = {} + for field, candidates in COLUMN_CANDIDATES.items(): + for candidate in candidates: + if candidate in available_columns: + selected[field] = candidate + break + + # A record with no duration column is "unknown", not "off": the + # timer state cannot be determined from this database version. + has_duration = "disappearing_mode_duration" in selected + if not has_duration: + self.log.warning( + "The ZDISAPPEARINGMODEDURATION column is not present in " + "this WhatsApp contacts database: disappearing messages " + "state is unknown" + ) + + columns = ["Z_PK"] + list(selected.values()) + cur.execute( + f"SELECT {', '.join(columns)} FROM ZWAADDRESSBOOKCONTACT;" + ) + fields = ["row_pk"] + list(selected.keys()) + + for row in cur: + record = dict(zip(fields, row)) + + for field in STRING_FIELDS: + if field in record: + record[field] = _decode_string(record[field]) + else: + record[field] = None + + for field in DATE_FIELDS: + if record.get(field) is not None: + record[field] = ( + convert_mactime_to_iso(record[field]) or None + ) + else: + record[field] = None + + duration = record.get("disappearing_mode_duration") + if has_duration: + record["disappearing_mode_is_on"] = bool(duration) + record["disappearing_mode_label"] = _label_duration( + duration + ) + else: + record["disappearing_mode_duration"] = None + record["disappearing_mode_is_on"] = None + record["disappearing_mode_label"] = None + + record.setdefault("phone_status", None) + record.setdefault("sync_policy", None) + + self.results.append(record) + finally: + cur.close() + conn.close() + + total_ephemeral = sum( + 1 for record in self.results if record["disappearing_mode_is_on"] + ) + self.log.info( + "Extracted a total of %d WhatsApp contacts (%d with disappearing " + "messages enabled)", + len(self.results), + total_ephemeral, + ) diff --git a/src/mvt/plugin.py b/src/mvt/plugin.py new file mode 100644 index 0000000..9257dc6 --- /dev/null +++ b/src/mvt/plugin.py @@ -0,0 +1,80 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Stable functions and modules which plugins can import. + +Anything else in mvt can be imported too, and may change +between releases without notice. +""" + +from mvt.android.modules.androidqf.base import AndroidQFModule +from mvt.android.modules.backup.base import BackupModule as AndroidBackupModule +from mvt.android.modules.bugreport.base import BugReportModule +from mvt.common.alerts import Alert, AlertLevel +from mvt.common.command import Command +from mvt.common.config import settings +from mvt.common.module import DatabaseCorruptedError, DatabaseNotFoundError, MVTModule +from mvt.common.module_loader import get_plugin_logger +from mvt.common.module_types import ( + ModuleAtomicResult, + ModuleResults, + ModuleSerializedResult, +) +from mvt.common.plugin_config import ( + MVTPluginSettings, + PluginConfigLoadError, + plugin_config_path, + plugin_data_folder, + plugin_env_prefix, +) +from mvt.common.utils import ( + convert_chrometime_to_datetime, + convert_datetime_to_iso, + convert_mactime_to_datetime, + convert_mactime_to_iso, + convert_unix_to_iso, + convert_unix_to_utc_datetime, +) +from mvt.common.version import MVT_VERSION +from mvt.ios.modules.base import IOSExtraction +from mvt.ios.modules.sysdiagnose.base import SysdiagnoseExtraction + +__all__ = [ + # Classes a plugin subclasses. + "MVTModule", + "IOSExtraction", + "SysdiagnoseExtraction", + "AndroidQFModule", + "AndroidBackupModule", + "BugReportModule", + "Command", + # Results and alerts. + "ModuleAtomicResult", + "ModuleResults", + "ModuleSerializedResult", + "Alert", + "AlertLevel", + # Errors a module raises. + "DatabaseNotFoundError", + "DatabaseCorruptedError", + # Settings. + "settings", + "MVTPluginSettings", + "PluginConfigLoadError", + "plugin_config_path", + "plugin_data_folder", + "plugin_env_prefix", + # Logging. + "get_plugin_logger", + # Timestamps. + "convert_chrometime_to_datetime", + "convert_datetime_to_iso", + "convert_mactime_to_datetime", + "convert_mactime_to_iso", + "convert_unix_to_iso", + "convert_unix_to_utc_datetime", + # MVT's version. + "MVT_VERSION", +] diff --git a/tests/android/test_artifact_tombstone_caused_by.py b/tests/android/test_artifact_tombstone_caused_by.py new file mode 100644 index 0000000..9a2df85 --- /dev/null +++ b/tests/android/test_artifact_tombstone_caused_by.py @@ -0,0 +1,54 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +"""A `Caused by:` line must not discard the whole text tombstone. + +Keys are matched as bare prefixes, so `Caused by: …` — an ordinary line inside +an abort message — reached the `Cause` key, failed the key comparison and +raised, which `Tombstones.run()` logged while dropping the entire crash record. +Seen on a 1.6 MB tombstone whose protobuf twin was zero bytes: the crash then +had no representation at all. +""" + +import datetime + +from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact + +TOMBSTONE = b"""\ +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +Build fingerprint: 'Xiaomi/vili_eea/vili:13/TKQ1.220829.002/V14.0.10.0:user/release-keys' +Revision: '0' +ABI: 'arm64' +Timestamp: 2023-08-24 14:54:47.999124034+0300 +Process uptime: 12199s +Cmdline: com.example.game +pid: 8044, tid: 26222, name: UnityMain >>> com.example.game <<< +uid: 10235 +signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr -------- +Abort message: 'No pending exception expected: java.lang.SecurityException: listen + at void android.os.Parcel.readException() (Parcel.java:2920) +Caused by: android.os.RemoteException: Remote stack trace: +\tat com.android.server.TelephonyRegistry.listen(TelephonyRegistry.java:1096) +""" + +WITH_CAUSE = TOMBSTONE + b"Cause: null pointer dereference\n" + + +class TestTombstoneCausedBy: + def _parse(self, content): + artifact = TombstoneCrashArtifact() + artifact.results = [] + artifact.parse("tombstone_23", datetime.datetime(2023, 8, 24), content) + return artifact.results + + def test_caused_by_line_does_not_discard_the_tombstone(self): + results = self._parse(TOMBSTONE) + assert len(results) == 1 + assert results[0]["pid"] == 8044 + assert results[0]["process_name"] == "UnityMain" + assert results[0]["uid"] == 10235 + + def test_the_real_cause_key_is_still_parsed(self): + results = self._parse(WITH_CAUSE) + assert results[0]["cause"] == "null pointer dereference" diff --git a/tests/android/test_check_backup_optional_failure.py b/tests/android/test_check_backup_optional_failure.py new file mode 100644 index 0000000..72dae42 --- /dev/null +++ b/tests/android/test_check_backup_optional_failure.py @@ -0,0 +1,39 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +"""An encrypted backup.ab must not take the whole check-androidqf run with it. + +`CmdAndroidCheckBackup.from_ab()` already raises `InvalidAndroidBackup` instead +of exiting when it runs as a sub-command (`check-androidqf` catches that and +skips the backup modules), for a wrong file format and for a parse error. The +password branches used to call `sys.exit(1)` unconditionally, which ends the +parent run inside `finish()` — before the intrusion-logs command and before the +timeline, alerts, urls, info and run-manifest are written. +""" + +import pytest + +from mvt.android.cmd_check_backup import CmdAndroidCheckBackup, InvalidAndroidBackup + +ENCRYPTED_AB_HEADER = b"ANDROID BACKUP\n5\n0\nAES-256\n" + b"\x00" * 64 + + +class TestCheckBackupOptionalFailure: + def _cmd(self, tmp_path, sub_command): + return CmdAndroidCheckBackup( + target_path=None, + results_path=str(tmp_path), + module_options={"interactive": False}, + sub_command=sub_command, + ) + + def test_missing_password_raises_when_nested(self, tmp_path): + cmd = self._cmd(tmp_path, sub_command=True) + with pytest.raises(InvalidAndroidBackup): + cmd.from_ab(ENCRYPTED_AB_HEADER) + + def test_missing_password_still_exits_on_its_own_command(self, tmp_path): + cmd = self._cmd(tmp_path, sub_command=False) + with pytest.raises(SystemExit): + cmd.from_ab(ENCRYPTED_AB_HEADER) diff --git a/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee b/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee new file mode 100644 index 0000000..1d27832 Binary files /dev/null and b/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee differ diff --git a/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d b/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d new file mode 100644 index 0000000..b8ae144 Binary files /dev/null and b/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d differ diff --git a/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f b/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f new file mode 100644 index 0000000..132059c Binary files /dev/null and b/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f differ diff --git a/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 b/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 new file mode 100644 index 0000000..d54a9c5 Binary files /dev/null and b/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 differ diff --git a/tests/common/test_cli_plugins.py b/tests/common/test_cli_plugins.py index ea7c7fa..5abb2c9 100644 --- a/tests/common/test_cli_plugins.py +++ b/tests/common/test_cli_plugins.py @@ -3,9 +3,12 @@ from types import SimpleNamespace import click from click.testing import CliRunner +from mvt.cli import cli as mvt_cli from mvt.common.cli_plugins import ( ANDROID_CLI_PLUGIN_GROUP, IOS_CLI_PLUGIN_GROUP, + MVT_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, BrokenPluginCommand, load_cli_commands_option, register_cli_commands_from_path, @@ -14,6 +17,9 @@ from mvt.common.cli_plugins import ( ) +# Keep the banner of the mvt group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + COMMAND_TEMPLATE = """ import click @@ -345,7 +351,11 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate( def entry_points(*, group): if group == IOS_CLI_PLUGIN_GROUP: return [_entry_point("ios-package", "ios_plugin:cli", ios_package)] - return [_entry_point("android-package", "android_plugin:cli", android_package)] + if group == ANDROID_CLI_PLUGIN_GROUP: + return [ + _entry_point("android-package", "android_plugin:cli", android_package) + ] + return [] monkeypatch.setattr( "mvt.common.cli_plugins.importlib.metadata.entry_points", @@ -369,3 +379,187 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate( assert set(ios_group.commands) == {"ios-file", "ios-package"} assert set(android_group.commands) == {"android-file", "android-package"} + + +def test_neutral_entry_point_group_is_not_registered_on_the_platform_clis( + monkeypatch, +): + @click.command() + def neutral_package(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point("neutral-package", "neutral_plugin:cli", neutral_package) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + ios_group = click.Group() + android_group = click.Group() + + register_cli_plugins( + ios_group, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable="TEST_IOS_COMMANDS", + ) + register_cli_plugins( + android_group, + entry_point_group=ANDROID_CLI_PLUGIN_GROUP, + environment_variable="TEST_ANDROID_COMMANDS", + ) + + assert not ios_group.commands + assert not android_group.commands + + +def test_environment_command_wins_collision_with_installed_command( + tmp_path, monkeypatch, caplog +): + command_path = _write_command( + tmp_path / "duplicate.py", + "duplicate", + message="environment command ran", + ) + + @click.command() + def installed_command(): + pass + + def entry_points(*, group): + if group == IOS_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "duplicate", + "ios_plugin:cli", + installed_command, + distribution="ios-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + monkeypatch.setenv("TEST_IOS_COMMANDS", str(command_path)) + group = click.Group() + + register_cli_plugins( + group, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable="TEST_IOS_COMMANDS", + ) + + assert group.commands["duplicate"] is not installed_command + result = CliRunner().invoke(group, ["duplicate"]) + assert result.exit_code == 0 + assert "environment command ran" in result.output + assert "the command name is already registered" in caplog.text + assert "ios-plugin 1.0 (ios_plugin:cli)" in caplog.text + + +def test_the_mvt_cli_gets_the_neutral_commands_and_no_platform_command( + monkeypatch, restore_cli_commands +): + @click.command() + def shared_package(): + click.echo("shared command ran") + + @click.command() + def ios_package(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [_entry_point("shared-package", "shared_plugin:cli", shared_package)] + if group == IOS_CLI_PLUGIN_GROUP: + return [_entry_point("ios-package", "ios_plugin:cli", ios_package)] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert "ios-package" not in mvt_cli.commands + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "shared-package"]) + assert result.exit_code == 0 + assert "shared command ran" in result.output + + +def test_builtin_mvt_command_wins_collision_with_neutral_command( + monkeypatch, caplog, restore_cli_commands +): + @click.command() + def neutral_version(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "version", + "neutral_plugin:cli", + neutral_version, + distribution="neutral-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + builtin_version = mvt_cli.commands["version"] + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert mvt_cli.commands["version"] is builtin_version + assert "the command name is already registered" in caplog.text + assert "neutral-plugin 1.0 (neutral_plugin:cli)" in caplog.text + + +def test_broken_neutral_plugin_does_not_break_the_mvt_cli( + monkeypatch, restore_cli_commands +): + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "broken", + "broken_plugin:cli", + exception=RuntimeError("missing dependency"), + distribution="broken-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert isinstance(mvt_cli.commands["broken"], BrokenPluginCommand) + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "version"]) + assert result.exit_code == 0 diff --git a/tests/common/test_cmd_check_iocs.py b/tests/common/test_cmd_check_iocs.py new file mode 100644 index 0000000..0c05c1b --- /dev/null +++ b/tests/common/test_cmd_check_iocs.py @@ -0,0 +1,225 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import json +import logging + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES +from mvt.common.cmd_check_iocs import CmdCheckIOCS +from mvt.common.module import MVTModule +from mvt.ios.cli import cli as ios_cli +from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES +from mvt.ios.modules.backup.manifest import Manifest + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + + +class CustomResultsModule(MVTModule): + """A custom module which declares the check-iocs pair of both platforms.""" + + slug = "custom_results" + supported_commands = ( + ("ios", "check-backup"), + ("ios", "check-iocs"), + ("android", "check-iocs"), + ) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BackupCheckerModule(MVTModule): + """An iOS module which implements check_indicators() without declaring check-iocs.""" + + slug = "backup_checker" + supported_commands = (("ios", "check-backup"),) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BugReportCheckerModule(MVTModule): + """The same for Android.""" + + slug = "bugreport_checker" + supported_commands = (("android", "check-bugreport"),) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BackupOnlyModule(MVTModule): + """A custom module which does not implement check_indicators().""" + + slug = "backup_only" + supported_commands = (("ios", "check-backup"),) + + def run(self) -> None: + pass + + +@pytest.mark.parametrize( + "platform, builtin_modules, checker_module", + [ + ("ios", IOS_CHECK_IOCS_MODULES, BackupCheckerModule), + ("android", ANDROID_CHECK_IOCS_MODULES, BugReportCheckerModule), + ], +) +def test_check_iocs_rechecks_the_stored_results_of_custom_modules( + platform, builtin_modules, checker_module, tmp_path, caplog +): + # check-iocs matches every .json in the results folder to the module + # with that slug, custom modules included, and runs its check_indicators() + # again over the stored results. + results = [{"domain": "example.org"}] + (tmp_path / "custom_results.json").write_text(json.dumps(results)) + (tmp_path / f"{checker_module.slug}.json").write_text(json.dumps(results)) + (tmp_path / "backup_only.json").write_text(json.dumps(results)) + CustomResultsModule.checked.clear() + checker_module.checked.clear() + + cmd = CmdCheckIOCS( + target_path=str(tmp_path), + custom_modules=[CustomResultsModule, checker_module, BackupOnlyModule], + platform=platform, + ) + cmd.modules = builtin_modules + + with caplog.at_level(logging.INFO): + cmd.run() + + # A module which declares the check-iocs pair is re-checked. + assert CustomResultsModule.checked == [results] + assert ( + 'Loading results from "custom_results.json" with module CustomResultsModule' + in caplog.text + ) + # So is a module which only implements check_indicators(). + assert checker_module.checked == [results] + # A module which does neither is not part of check-iocs. + assert "backup_only.json" not in caplog.text + + +@pytest.mark.parametrize( + "platform, builtin_modules, listed, not_listed", + [ + ( + "ios", + IOS_CHECK_IOCS_MODULES, + "BackupCheckerModule", + "BugReportCheckerModule", + ), + ( + "android", + ANDROID_CHECK_IOCS_MODULES, + "BugReportCheckerModule", + "BackupCheckerModule", + ), + ], +) +def test_check_iocs_lists_the_custom_modules_it_runs( + platform, builtin_modules, listed, not_listed, caplog +): + cmd = CmdCheckIOCS( + custom_modules=[ + CustomResultsModule, + BackupCheckerModule, + BugReportCheckerModule, + BackupOnlyModule, + ], + platform=platform, + ) + cmd.modules = builtin_modules + + with caplog.at_level(logging.INFO): + cmd.list_modules() + + assert "CustomResultsModule" in caplog.text + # The module which implements check_indicators() for this platform is listed. + assert listed in caplog.text + # The one for the other platform is not, and neither is BackupOnlyModule. + assert not_listed not in caplog.text + assert "BackupOnlyModule" not in caplog.text + + +class ReplacementManifest(Manifest): + """A replacement for a built-in module which does not declare check-iocs.""" + + supported_commands = (("ios", "check-backup"),) + replaces = Manifest + + +def test_check_iocs_uses_a_replacement_of_a_built_in_module(): + # A replacement which subclasses a built-in module inherits its + # check_indicators(). check-iocs then runs it in place of that module. + cmd = CmdCheckIOCS(custom_modules=[ReplacementManifest], platform="ios") + cmd.modules = IOS_CHECK_IOCS_MODULES + + available = cmd._available_modules() + + assert ReplacementManifest in available + assert Manifest not in available + + +LOADED_MODULE = ''' +from mvt.common.module import MVTModule + + +class LoadedResultsModule(MVTModule): + """A module loaded from a file with --load-module.""" + + slug = "loaded_results" + supported_commands = (("ios", "check-iocs"), ("android", "check-iocs")) + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.log.warning("loaded module checked %d results", len(self.results)) +''' + + +@pytest.mark.parametrize("cli", [ios_cli, android_cli], ids=["mvt-ios", "mvt-android"]) +def test_check_iocs_loads_custom_modules_from_a_file_on_each_cli(cli, tmp_path, caplog): + module_path = tmp_path / "loaded_module.py" + module_path.write_text(LOADED_MODULE) + results_folder = tmp_path / "results" + results_folder.mkdir() + (results_folder / "loaded_results.json").write_text(json.dumps([{"a": 1}])) + + with caplog.at_level(logging.INFO): + result = CliRunner().invoke( + cli, + [ + *OFFLINE, + "check-iocs", + "--load-module", + str(module_path), + str(results_folder), + ], + ) + + assert result.exit_code == 0, result.output + assert "loaded module checked 1 results" in caplog.text diff --git a/tests/common/test_cmd_plugins.py b/tests/common/test_cmd_plugins.py new file mode 100644 index 0000000..adf9322 --- /dev/null +++ b/tests/common/test_cmd_plugins.py @@ -0,0 +1,236 @@ +import json +from types import SimpleNamespace + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.common.cmd_plugins import plugins +from mvt.common.module import MVTModule +from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP +from mvt.common.updates import PluginUpdates +from mvt.ios.cli import cli as ios_cli + + +class ExampleModule(MVTModule): + pass + + +class AnotherModule(MVTModule): + pass + + +class FakeDistribution: + def __init__(self, name, version="1.0.0", direct_url=None): + self.name = name + self.version = version + self.direct_url = direct_url + + def read_text(self, file_name): + if file_name == "direct_url.json" and self.direct_url is not None: + return json.dumps(self.direct_url) + return None + + +def _entry_point(name, distribution, modules=None, exception=None): + def load(): + if exception is not None: + raise exception + return modules + + return SimpleNamespace( + name=name, value="example_plugin:modules", dist=distribution, load=load + ) + + +def _run(command, arguments): + # Keep rich from wrapping the table while its content is being asserted. + return CliRunner().invoke(command, arguments, env={"COLUMNS": "200"}) + + +def _table_rows(output): + """Return the content of the table rows, without the header and the box.""" + return [ + [cell.strip() for cell in line.strip().strip("│").split("│")] + for line in output.splitlines() + if "│" in line + ] + + +def _table_header(output): + for line in output.splitlines(): + if "┃" in line: + return [cell.strip() for cell in line.strip().strip("┃").split("┃")] + return [] + + +def _install(monkeypatch, distributions, entry_points): + monkeypatch.setattr( + "mvt.common.cmd_plugins.installed_plugin_distributions", + lambda: distributions, + ) + monkeypatch.setattr( + "mvt.common.cmd_plugins.importlib.metadata.entry_points", + lambda *, group: entry_points.get(group, []), + ) + + +def test_plugins_is_a_builtin_command_of_the_mvt_cli_only(): + assert mvt_cli.commands["plugins"] is plugins + assert "plugins" not in ios_cli.commands + assert "plugins" not in android_cli.commands + + +def test_list_shows_what_every_plugin_contributes(monkeypatch): + index_plugin = FakeDistribution("example-plugin", version="1.2.0") + repository_plugin = FakeDistribution( + "repository-plugin", + version="0.1.0", + direct_url={ + "url": "https://example.org/plugin.git", + "vcs_info": {"vcs": "git", "commit_id": "b" * 40}, + }, + ) + local_plugin = FakeDistribution( + "local-plugin", + direct_url={"url": "file:///plugins", "dir_info": {"editable": True}}, + ) + _install( + monkeypatch, + [index_plugin, local_plugin, repository_plugin], + { + MODULES_ENTRY_POINT_GROUP: [ + _entry_point( + "example", index_plugin, modules=[ExampleModule, AnotherModule] + ), + _entry_point("local", local_plugin, modules=lambda: [ExampleModule]), + ], + IOS_CLI_PLUGIN_GROUP: [_entry_point("summarize", repository_plugin)], + ANDROID_CLI_PLUGIN_GROUP: [_entry_point("triage", local_plugin)], + NEUTRAL_CLI_PLUGIN_GROUP: [_entry_point("report", repository_plugin)], + }, + ) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + # Plugins are listed by name, with the modules and the commands each of + # them contributes. + assert _table_header(result.output) == [ + "Name", + "Version", + "Origin", + "Modules", + "Commands", + ] + assert _table_rows(result.output) == [ + ["example-plugin", "1.2.0", "pypi", "2", "-"], + ["local-plugin", "1.0.0", "local", "1", "triage"], + ["repository-plugin", "0.1.0", "git+bbbbbbbb", "0", "report, summarize"], + ] + + +def test_list_reports_a_broken_module_entry_point(monkeypatch): + plugin = FakeDistribution("broken-plugin") + _install( + monkeypatch, + [plugin], + { + MODULES_ENTRY_POINT_GROUP: [ + _entry_point( + "broken", plugin, exception=ImportError("missing dependency") + ) + ] + }, + ) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + assert _table_rows(result.output) == [ + ["broken-plugin", "1.0.0", "pypi", "error", "-"] + ] + + +def test_list_without_plugins(monkeypatch): + _install(monkeypatch, [], {}) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + assert result.output == "No MVT plugins are installed.\n" + + +def test_check_updates_prints_the_findings_and_ignores_the_throttle(monkeypatch): + findings = [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ] + _install(monkeypatch, [FakeDistribution("example-plugin")], {}) + monkeypatch.setattr(PluginUpdates, "check", lambda self: findings) + monkeypatch.setattr( + PluginUpdates, + "should_check", + lambda self: pytest.fail("an explicit check must not be throttled"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "Plugin updates available:" in result.output + assert "example-plugin 1.0.0 → 1.2.0" in result.output + assert "Upgrade with: pip install -U example-plugin" in result.output + assert "MVT does not install plugin updates." in result.output + + +def test_check_updates_without_available_updates(monkeypatch): + _install(monkeypatch, [FakeDistribution("example-plugin")], {}) + monkeypatch.setattr(PluginUpdates, "check", lambda self: []) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "All plugins are up to date." in result.output + + +def test_check_updates_without_plugins(monkeypatch): + _install(monkeypatch, [], {}) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("nothing must be checked without plugins"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "No MVT plugins are installed." in result.output + + +def test_check_updates_without_network_access(monkeypatch): + monkeypatch.setattr("mvt.common.cmd_plugins.settings.NETWORK_ACCESS_ALLOWED", False) + monkeypatch.setattr( + "mvt.common.cmd_plugins.installed_plugin_distributions", + lambda: pytest.fail("plugins must not be listed without network access"), + ) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("nothing must be checked without network access"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "Network access is disabled" in result.output diff --git a/tests/common/test_command.py b/tests/common/test_command.py index 4dbfe1a..8e09f57 100644 --- a/tests/common/test_command.py +++ b/tests/common/test_command.py @@ -68,6 +68,123 @@ class CustomDependsOnBuiltin(RecordingModule): dependencies = (FirstModule,) +class ReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + def run(self): + super().run() + self.results = ["replacement"] + + +class OtherReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + +class UnrelatedReplacementModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = IndependentModule + + +class ReplacementOfReplacementModule(ReplacementModule): + supported_commands = (("ios", "check-backup"),) + replaces = ReplacementModule + + def run(self): + super().run() + self.results = ["replacement of replacement"] + + +class ReplacesOwnDependencyModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + dependencies = (FirstModule,) + + +class DisabledReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + enabled = False + + +class MutualReplacementOne(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class MutualReplacementTwo(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = MutualReplacementOne + + +MutualReplacementOne.replaces = MutualReplacementTwo + + +class CycleOneModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class CycleTwoModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = CycleOneModule + + +class CycleThreeModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = CycleTwoModule + + +CycleOneModule.replaces = CycleThreeModule + + +class UnavailableDependencyModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class ReplacementMissingDependency(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + dependencies = (UnavailableDependencyModule,) + + +class SharedSlugModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + slug = "shared_slug" + + +class OtherSharedSlugModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + slug = "shared_slug" + + +class ReplacementKeepingTheSlug(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + slug = "first_module" + + +class SameNameReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + def run(self): + super().run() + self.results = ["replacement"] + + +# Modules replacing a built-in one usually keep its class name, which is the +# name `--module` matches on. +SameNameReplacementModule.__name__ = "FirstModule" + + +def logged_substitutions(caplog) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.INFO and "replaces module" in record.getMessage() + ] + + class RecordingCommand(Command): def init(self): self.initialized = True @@ -157,7 +274,7 @@ class TestCommand: assert not hasattr(cmd, "initialized") assert "Circular module dependency detected" in caplog.text - def test_unavailable_dependency_warns_and_stops(self, caplog): + def test_unavailable_dependency_only_skips_the_dependent_module(self, caplog): class UnavailableModule(RecordingModule): pass @@ -165,14 +282,97 @@ class TestCommand: dependencies = (UnavailableModule,) cmd = RecordingCommand() - cmd.modules = [DependentModule] + cmd.modules = [DependentModule, IndependentModule, FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "FirstModule"] + assert cmd.initialized + assert "Module DependentModule will be SKIPPED" in caplog.text + assert "depends on module UnavailableModule" in caplog.text + + def test_modules_depending_on_a_skipped_module_are_skipped_too(self, caplog): + class UnavailableModule(RecordingModule): + pass + + class SkippedModule(RecordingModule): + dependencies = (UnavailableModule,) + + class DependsOnSkippedModule(RecordingModule): + dependencies = (SkippedModule,) + + class DependsOnTheChain(RecordingModule): + dependencies = (DependsOnSkippedModule,) + + cmd = RecordingCommand() + cmd.modules = [ + DependsOnTheChain, + DependsOnSkippedModule, + SkippedModule, + IndependentModule, + ] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule"] + skip_warnings = [ + record.getMessage() + for record in caplog.records + if "will be SKIPPED" in record.getMessage() + ] + assert len(skip_warnings) == 3 + assert [warning.split()[1] for warning in skip_warnings] == [ + "SkippedModule", + "DependsOnSkippedModule", + "DependsOnTheChain", + ] + # Every warning names the root cause: the module missing a dependency + # and the dependency it is missing. + assert all("UnavailableModule" in warning for warning in skip_warnings) + assert all("module SkippedModule" in warning for warning in skip_warnings[1:]) + + def test_explicitly_selected_module_with_missing_dependency_runs_nothing( + self, caplog + ): + class UnavailableModule(RecordingModule): + pass + + class DependentModule(RecordingModule): + dependencies = (UnavailableModule,) + + cmd = RecordingCommand(module_name="DependentModule") + cmd.modules = [DependentModule, IndependentModule] with caplog.at_level(logging.WARNING): cmd.run() assert RecordingModule.run_order == [] - assert not hasattr(cmd, "initialized") - assert "depends on unavailable module UnavailableModule" in caplog.text + assert "Module DependentModule will be SKIPPED" in caplog.text + assert "No modules will be run" in caplog.text + # Nothing else was selected, so the warning must not promise that the + # analysis continues right before saying that it does not. + assert "The rest of the analysis will still run" not in caplog.text + + def test_unaffected_dependency_chains_keep_their_order(self, caplog): + class UnavailableModule(RecordingModule): + pass + + class SkippedModule(RecordingModule): + dependencies = (UnavailableModule,) + + cmd = RecordingCommand() + cmd.modules = [ThirdModule, SkippedModule, SecondModule, FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "FirstModule", + "SecondModule", + "ThirdModule", + ] def test_custom_modules_are_filtered_before_ordering(self): cmd = RecordingCommand() @@ -220,3 +420,376 @@ class TestCommand: cmd.run() assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"] + + def test_custom_module_replaces_builtin(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"] + assert cmd.module_replacements == {FirstModule: ReplacementModule} + assert ( + "Module ReplacementModule from" in caplog.text + and "replaces module FirstModule" in caplog.text + ) + + def test_modules_without_replaces_all_run(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [CustomIOSBackupModule] + + cmd.run() + + assert RecordingModule.run_order == ["FirstModule", "CustomIOSBackupModule"] + assert cmd.module_replacements == {} + + def test_unavailable_replaced_module_is_ignored(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"] + assert cmd.module_replacements == {} + assert "replaces module FirstModule" not in caplog.text + + def test_dependencies_are_resolved_to_the_replacement(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [ReplacementModule] + + cmd.run() + + assert RecordingModule.run_order == ["ReplacementModule", "SecondModule"] + second = next( + module for module in cmd.executed if isinstance(module, SecondModule) + ) + assert isinstance(second.dependency_modules[FirstModule], ReplacementModule) + assert second.results == ["replacement", "second"] + + def test_replacement_with_unavailable_dependency_skips_its_dependents( + self, caplog + ): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, SecondModule, IndependentModule] + cmd.custom_modules = [ReplacementMissingDependency] + + with caplog.at_level(logging.INFO): + cmd.run() + + # The replacement cannot run, and the module it replaced was dropped + # from the run by the replacement, so neither of them produces + # results and the module depending on the replaced one is skipped. + assert RecordingModule.run_order == ["IndependentModule"] + assert ( + "Module ReplacementMissingDependency will be SKIPPED: it depends " + "on module UnavailableDependencyModule, which is not available in " + "this command." in caplog.text + ) + # The skipped dependent is told which module it actually depends on, + # and the module class its author declared. + assert ( + "Module SecondModule will be SKIPPED: it depends on module " + "ReplacementMissingDependency (replacing module FirstModule), " + "itself skipped for depending on unavailable module " + "UnavailableDependencyModule." in caplog.text + ) + + def test_selected_replacement_with_unavailable_dependency_runs_nothing( + self, caplog, tmp_path + ): + cmd = RecordingCommand(module_name="FirstModule", results_path=str(tmp_path)) + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementMissingDependency] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert ( + "Module FirstModule was replaced by module " + "ReplacementMissingDependency, which is run in its place." + in caplog.text + ) + assert "Module ReplacementMissingDependency will be SKIPPED" in caplog.text + assert "No modules will be run" in caplog.text + # Nothing else was selected, so the warnings must not promise that the + # analysis continues right before saying that it does not. + assert "The rest of the analysis will still run" not in caplog.text + # No module ran, so no results were stored next to the command log. + assert [path.name for path in tmp_path.iterdir()] == ["command.log"] + + def test_chained_replacements_are_resolved(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [ReplacementModule, ReplacementOfReplacementModule] + + cmd.run() + + assert RecordingModule.run_order == [ + "ReplacementOfReplacementModule", + "SecondModule", + ] + second = next( + module for module in cmd.executed if isinstance(module, SecondModule) + ) + assert second.results == ["replacement of replacement", "second"] + + def test_replacement_which_is_not_a_subclass_warns(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [IndependentModule] + cmd.custom_modules = [UnrelatedReplacementModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["UnrelatedReplacementModule"] + assert "is not a subclass of it" in caplog.text + + def test_multiple_modules_replacing_the_same_module_warn(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementModule, OtherReplacementModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "ReplacementModule", + "OtherReplacementModule", + ] + assert cmd.module_replacements == {FirstModule: ReplacementModule} + assert ( + "Modules ReplacementModule and OtherReplacementModule both replace " + "module FirstModule. Both of them will run, FirstModule will not, " + "and modules depending on FirstModule will use the results of " + "ReplacementModule." in caplog.text + ) + assert "overwrite each other's results file" in caplog.text + + def test_module_replacing_its_own_dependency_runs(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacesOwnDependencyModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["ReplacesOwnDependencyModule"] + assert ( + "Module ReplacesOwnDependencyModule depends on module FirstModule, " + "which it also replaces" in caplog.text + ) + + def test_literal_self_dependency_is_still_circular(self, caplog): + class SelfDependentModule(RecordingModule): + pass + + SelfDependentModule.dependencies = (SelfDependentModule,) + + cmd = RecordingCommand() + cmd.modules = [SelfDependentModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert "Circular module dependency detected" in caplog.text + + def test_disabled_replacement_keeps_the_replaced_module(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [DisabledReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["FirstModule"] + assert cmd.module_replacements == {} + assert logged_substitutions(caplog) == [] + + def test_modules_replacing_each_other_all_run(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [MutualReplacementOne, MutualReplacementTwo] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [ + "MutualReplacementOne", + "MutualReplacementTwo", + ] + assert cmd.module_replacements == {} + assert ( + "Modules MutualReplacementOne, MutualReplacementTwo replace each " + "other in a cycle" in caplog.text + ) + assert logged_substitutions(caplog) == [] + + def test_replacement_cycle_of_three_modules_all_run(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [CycleOneModule, CycleTwoModule, CycleThreeModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [ + "CycleOneModule", + "CycleTwoModule", + "CycleThreeModule", + ] + assert cmd.module_replacements == {} + assert ( + "Modules CycleOneModule, CycleThreeModule, CycleTwoModule replace " + "each other in a cycle" in caplog.text + ) + assert logged_substitutions(caplog) == [] + + def test_selected_replaced_module_name_runs_the_replacement(self, caplog): + cmd = RecordingCommand(module_name="FirstModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["ReplacementModule"] + assert ( + "Module FirstModule was replaced by module ReplacementModule" + in caplog.text + ) + + def test_unknown_selected_module_warns_and_stops(self, caplog): + cmd = RecordingCommand(module_name="NoSuchModule") + cmd.name = "check-backup" + cmd.modules = [FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert ( + "No module named NoSuchModule is available for the check-backup " + "command" in caplog.text + ) + + def test_selected_module_name_matches_the_replacement(self): + cmd = RecordingCommand(module_name="FirstModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [SameNameReplacementModule] + + cmd.run() + + assert len(cmd.executed) == 1 + assert isinstance(cmd.executed[0], SameNameReplacementModule) + assert cmd.executed[0].results == ["replacement"] + + def test_list_modules_reflects_replacements(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.list_modules() + + listed = [ + record.getMessage() + for record in caplog.records + if "Modules from" in record.getMessage() + ] + assert any("ReplacementModule" in message for message in listed) + assert not any("FirstModule" in message for message in listed) + + def test_modules_sharing_a_slug_are_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [SharedSlugModule, OtherSharedSlugModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "SharedSlugModule", + "OtherSharedSlugModule", + ] + collisions = [ + record.getMessage() + for record in caplog.records + if "both use the slug" in record.getMessage() + ] + assert len(collisions) == 1 + assert "Modules SharedSlugModule from" in collisions[0] + assert "and OtherSharedSlugModule from" in collisions[0] + assert "both use the slug shared_slug" in collisions[0] + assert "overwrites the results of the other in shared_slug.json" in ( + collisions[0] + ) + + def test_replacement_keeping_the_replaced_slug_is_not_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementKeepingTheSlug] + + with caplog.at_level(logging.WARNING): + cmd.run() + + # The replaced module is no longer part of the run, so taking over its + # slug is what the replacement is for, not a collision. + assert RecordingModule.run_order == ["ReplacementKeepingTheSlug"] + assert ReplacementKeepingTheSlug.get_slug() == FirstModule.get_slug() + assert "both use the slug" not in caplog.text + + def test_modules_with_distinct_slugs_are_not_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [CustomIOSBackupModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert "both use the slug" not in caplog.text diff --git a/tests/common/test_command_modules.py b/tests/common/test_command_modules.py new file mode 100644 index 0000000..8f79ce9 --- /dev/null +++ b/tests/common/test_command_modules.py @@ -0,0 +1,27 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES +from mvt.android.modules.androidqf import ANDROIDQF_MODULES +from mvt.android.modules.backup import BACKUP_MODULES as ANDROID_BACKUP_MODULES +from mvt.android.modules.bugreport import BUGREPORT_MODULES +from mvt.android.modules.intrusion_logs import INTRUSION_LOGS_MODULES +from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES +from mvt.ios.modules.backup import BACKUP_MODULES as IOS_BACKUP_MODULES +from mvt.ios.modules.fs import FS_MODULES +from mvt.ios.modules.mixed import MIXED_MODULES + + +def test_the_check_iocs_lists_are_the_families_of_their_platform(): + # The CLI reads these same lists, so nothing composing one elsewhere can + # drift from what the command runs. This pins what the lists are composed + # of. + assert IOS_CHECK_IOCS_MODULES == IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES + assert ANDROID_CHECK_IOCS_MODULES == ( + ANDROID_BACKUP_MODULES + + BUGREPORT_MODULES + + ANDROIDQF_MODULES + + INTRUSION_LOGS_MODULES + ) diff --git a/tests/common/test_module_loader.py b/tests/common/test_module_loader.py index 15b6052..527d01c 100644 --- a/tests/common/test_module_loader.py +++ b/tests/common/test_module_loader.py @@ -1,12 +1,19 @@ +from pathlib import Path + import pytest +from mvt.common.cli_plugins import _module_name_for_path as _command_module_name from mvt.common.module import MVTModule from mvt.common.module_loader import ( CustomModuleLoadError, + _module_name_for_path, + get_module_logger, + get_plugin_logger, load_custom_modules, load_custom_modules_from_path, module_supports_command, ) +from mvt.ios.modules.mixed.whatsapp import Whatsapp MODULE_TEMPLATE = """ @@ -144,3 +151,66 @@ def test_module_supports_command_honors_supported_commands(tmp_path): assert module_supports_command(module, "ios", "check-backup") assert not module_supports_command(module, "ios", "check-fs") + + +def test_get_module_logger_keeps_builtin_names(): + assert get_module_logger(Whatsapp).name == "mvt.ios.modules.mixed.whatsapp" + + +def test_get_module_logger_parents_package_modules_under_mvt_ext(): + class PackageModule(MVTModule): + pass + + PackageModule.__module__ = "some_plugin_package.ios.custom" + + assert ( + get_module_logger(PackageModule).name + == "mvt.ext.some_plugin_package.ios.custom" + ) + + +def test_get_module_logger_strips_the_plugin_package_prefix(): + class PluginModule(MVTModule): + pass + + PluginModule.__module__ = "mvt_plugin_example_org.ios.custom" + + assert get_module_logger(PluginModule).name == "mvt.ext.example_org.ios.custom" + + +def test_get_module_logger_only_strips_the_prefix_from_the_top_level(): + class NestedModule(MVTModule): + pass + + NestedModule.__module__ = "other_package.mvt_plugin_sub" + + assert get_module_logger(NestedModule).name == "mvt.ext.other_package.mvt_plugin_sub" + + +def test_get_module_logger_names_path_modules_after_their_file(tmp_path): + module_path = _write_module(tmp_path / "my_custom_module.py", "PathModule") + module = load_custom_modules_from_path(str(module_path))[0] + + assert get_module_logger(module).name == "mvt.ext.my_custom_module" + + +def test_get_plugin_logger_uses_the_same_namespace_as_modules(): + assert ( + get_plugin_logger("mvt_plugin_example_org.commands.summarize").name + == "mvt.ext.example_org.commands.summarize" + ) + assert get_plugin_logger("example_plugin.cli").name == "mvt.ext.example_plugin.cli" + + +def test_get_plugin_logger_keeps_builtin_names(): + assert get_plugin_logger("mvt.ios.cli").name == "mvt.ios.cli" + + +def test_get_plugin_logger_names_loaded_files_after_the_file(): + # A file loaded with --load-command or --load-module is imported under a + # mangled name. The log names the file instead. + command_name = _command_module_name(Path("/tmp/case_summary.py")) + module_name = _module_name_for_path(Path("/tmp/my_custom_module.py")) + + assert get_plugin_logger(command_name).name == "mvt.ext.case_summary" + assert get_plugin_logger(module_name).name == "mvt.ext.my_custom_module" diff --git a/tests/common/test_plugin.py b/tests/common/test_plugin.py new file mode 100644 index 0000000..255f2e8 --- /dev/null +++ b/tests/common/test_plugin.py @@ -0,0 +1,34 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import mvt.plugin +from mvt.android.modules.backup.base import BackupModule +from mvt.common.config import settings + +from ..plugin_fixtures import run_isolated_python + + +def test_the_exported_names_are_the_public_names(): + public = {name for name in vars(mvt.plugin) if not name.startswith("_")} + + assert public == set(mvt.plugin.__all__) + assert mvt.plugin.settings is settings + assert mvt.plugin.AndroidBackupModule is BackupModule + + +def test_the_surface_imports_before_anything_else_of_mvt(tmp_path): + # A plugin can import the surface as its first import of MVT. The + # subprocess gets a temporary home because importing MVT writes its + # configuration file. + result = run_isolated_python( + "from mvt.plugin import IOSExtraction, MVT_VERSION, settings\n" + "assert MVT_VERSION\n" + "assert settings.NETWORK_TIMEOUT > 0\n" + "assert IOSExtraction.__name__ == 'IOSExtraction'\n", + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + assert result.stderr == "" diff --git a/tests/common/test_plugin_config.py b/tests/common/test_plugin_config.py new file mode 100644 index 0000000..9bcf9eb --- /dev/null +++ b/tests/common/test_plugin_config.py @@ -0,0 +1,391 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import stat +import sys +from typing import Optional + +import pytest +import yaml + +from mvt.common.plugin_config import ( + MVTPluginSettings, + PluginConfigLoadError, + plugin_config_folder, + plugin_config_path, + plugin_data_folder, + plugin_env_prefix, +) + + +class ExamplePluginSettings(MVTPluginSettings): + plugin_name = "example-plugin" + + API_KEY: Optional[str] = None + CACHE_FOLDER: str = "cache" + MAX_RESULTS: int = 25 + + +class OtherPluginSettings(MVTPluginSettings): + plugin_name = "other-plugin" + + API_KEY: Optional[str] = None + + +@pytest.fixture +def config_folder(tmp_path, monkeypatch): + monkeypatch.setattr( + "mvt.common.plugin_config.user_config_dir", + lambda *args, **kwargs: str(tmp_path), + ) + return tmp_path + + +@pytest.fixture +def data_folder(tmp_path, monkeypatch): + folder = tmp_path / "data" + monkeypatch.setattr( + "mvt.common.plugin_config.user_data_dir", + lambda *args, **kwargs: str(folder), + ) + return folder + + +def _write_plugin_file(plugin_name, values): + config_path = plugin_config_path(plugin_name) + os.makedirs(os.path.dirname(config_path), exist_ok=True) + content = values if isinstance(values, str) else yaml.dump(values) + with open(config_path, "w") as config_file: + config_file.write(content) + return config_path + + +def test_plugin_paths_and_prefixes_are_namespaced(config_folder): + assert plugin_config_folder() == str(config_folder / "plugins") + assert plugin_config_path("example-plugin") == str( + config_folder / "plugins" / "example-plugin.yaml" + ) + assert plugin_env_prefix("example-plugin") == "MVT_PLUGIN_EXAMPLE_PLUGIN_" + assert plugin_env_prefix("other-plugin") == "MVT_PLUGIN_OTHER_PLUGIN_" + + +def test_defaults_are_used_without_file_or_environment(config_folder): + settings = ExamplePluginSettings.load() + + assert settings.API_KEY is None + assert settings.CACHE_FOLDER == "cache" + assert settings.MAX_RESULTS == 25 + assert not os.path.exists(plugin_config_path("example-plugin")) + + +def test_values_are_loaded_from_the_plugin_file(config_folder): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-file" + assert settings.MAX_RESULTS == 5 + assert settings.CACHE_FOLDER == "cache" + + +def test_environment_overrides_the_plugin_file(config_folder, monkeypatch): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment") + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-environment" + assert settings.MAX_RESULTS == 5 + + +def test_arguments_override_the_environment_and_the_plugin_file( + config_folder, monkeypatch +): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment") + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "10") + + settings = ExamplePluginSettings(API_KEY="from-argument") + + assert settings.API_KEY == "from-argument" + assert settings.MAX_RESULTS == 10 + + +def test_save_and_load_round_trip(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + settings.MAX_RESULTS = 100 + + settings.save() + + config_path = plugin_config_path("example-plugin") + assert os.path.isfile(config_path) + with open(config_path) as config_file: + assert yaml.safe_load(config_file) == { + "API_KEY": "saved-key", + "MAX_RESULTS": 100, + } + + reloaded = ExamplePluginSettings.load() + assert reloaded.API_KEY == "saved-key" + assert reloaded.MAX_RESULTS == 100 + assert reloaded.CACHE_FOLDER == "cache" + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_saved_file_is_only_readable_by_the_user(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + + settings.save() + + config_path = plugin_config_path("example-plugin") + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 + folder_mode = stat.S_IMODE(os.stat(plugin_config_folder()).st_mode) + assert folder_mode & 0o077 == 0 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_save_restricts_the_permissions_of_an_existing_file(config_folder): + config_path = _write_plugin_file("example-plugin", {"API_KEY": "from-file"}) + os.chmod(config_path, 0o644) + + settings = ExamplePluginSettings.load() + settings.MAX_RESULTS = 100 + settings.save() + + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 + assert os.listdir(plugin_config_folder()) == ["example-plugin.yaml"] + + +def test_save_only_persists_non_default_values(config_folder): + settings = ExamplePluginSettings.load() + settings.CACHE_FOLDER = "another-cache" + + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"CACHE_FOLDER": "another-cache"} + + +def test_save_does_not_persist_values_coming_from_the_environment( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + + settings = ExamplePluginSettings.load() + assert settings.API_KEY == "environment-secret" + settings.MAX_RESULTS = 100 + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"MAX_RESULTS": 100} + + +def test_an_invalid_environment_variable_still_protects_the_other_values( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "not-an-int") + + settings = ExamplePluginSettings(MAX_RESULTS=100) + assert settings.API_KEY == "environment-secret" + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + saved_values = yaml.safe_load(config_file) + assert saved_values == {"MAX_RESULTS": 100} + assert "API_KEY" not in saved_values + + +def test_save_persists_values_which_differ_from_the_environment( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + + settings = ExamplePluginSettings.load() + settings.API_KEY = "chosen-key" + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"API_KEY": "chosen-key"} + + +def test_save_does_not_write_the_mvt_configuration_file(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + + settings.save() + + assert os.listdir(config_folder) == ["plugins"] + + +def test_unknown_keys_in_the_plugin_file_are_ignored(config_folder): + _write_plugin_file( + "example-plugin", + {"API_KEY": "from-file", "UNKNOWN_SETTING": "ignored"}, + ) + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-file" + assert not hasattr(settings, "UNKNOWN_SETTING") + + +def test_unparsable_plugin_file_is_reported_with_its_path(config_folder): + config_path = _write_plugin_file("example-plugin", "API_KEY: [unclosed\n") + + with pytest.raises(PluginConfigLoadError) as raised: + ExamplePluginSettings.load() + + assert config_path in str(raised.value) + + +def test_plugin_file_which_is_not_a_mapping_is_reported_with_its_path(config_folder): + config_path = _write_plugin_file("example-plugin", "- one\n- two\n") + + with pytest.raises(PluginConfigLoadError) as raised: + ExamplePluginSettings.load() + + assert config_path in str(raised.value) + assert "mapping of setting names" in str(raised.value) + + +def test_plugins_do_not_interfere_with_each_other(config_folder, monkeypatch): + _write_plugin_file("other-plugin", {"API_KEY": "other-file-key"}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "example-environment-key") + + example_settings = ExamplePluginSettings.load() + other_settings = OtherPluginSettings.load() + + assert example_settings.API_KEY == "example-environment-key" + assert other_settings.API_KEY == "other-file-key" + + example_settings.MAX_RESULTS = 100 + example_settings.save() + assert sorted(os.listdir(plugin_config_folder())) == [ + "example-plugin.yaml", + "other-plugin.yaml", + ] + with open(plugin_config_path("other-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"API_KEY": "other-file-key"} + + +def test_subclass_without_plugin_name_is_rejected(): + with pytest.raises(TypeError, match="plugin_name"): + + class MissingNameSettings(MVTPluginSettings): + API_KEY: Optional[str] = None + + +def test_subclass_with_invalid_plugin_name_is_rejected(): + with pytest.raises(ValueError, match="Invalid plugin name"): + + class InvalidNameSettings(MVTPluginSettings): + plugin_name = "Bad/Name" + + +def test_underscores_are_not_allowed_in_plugin_names(): + # Underscores are replaced by dashes in the environment prefix, so allowing + # both would let two plugin names share one environment namespace. + with pytest.raises(ValueError, match="Invalid plugin name"): + + class UnderscoreNameSettings(MVTPluginSettings): + plugin_name = "under_score" + + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_config_path("under_score") + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_env_prefix("under_score") + + +@pytest.mark.parametrize( + "plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""] +) +def test_unsafe_plugin_names_have_no_configuration_path(plugin_name): + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_config_path(plugin_name) + + +def test_data_folder_is_namespaced_and_created(data_folder): + folder = plugin_data_folder("example-plugin") + + assert folder == str(data_folder / "plugin-data" / "example-plugin") + assert os.path.isdir(folder) + + +def test_data_folder_can_be_requested_repeatedly(data_folder): + folder = plugin_data_folder("example-plugin") + with open(os.path.join(folder, "kept.json"), "w") as data_file: + data_file.write("{}") + + assert plugin_data_folder("example-plugin") == folder + assert os.listdir(folder) == ["kept.json"] + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_data_folder_is_only_accessible_by_the_user(data_folder): + folder = plugin_data_folder("example-plugin") + + assert stat.S_IMODE(os.stat(folder).st_mode) & 0o077 == 0 + parent_mode = stat.S_IMODE(os.stat(os.path.dirname(folder)).st_mode) + assert parent_mode & 0o077 == 0 + + +def test_plugins_get_their_own_data_folder(data_folder): + example_folder = plugin_data_folder("example-plugin") + other_folder = plugin_data_folder("other-plugin") + + assert example_folder != other_folder + assert sorted(os.listdir(data_folder / "plugin-data")) == [ + "example-plugin", + "other-plugin", + ] + + +def test_data_folder_does_not_touch_the_configuration_folder( + config_folder, data_folder +): + plugin_data_folder("example-plugin") + + assert not os.path.exists(plugin_config_folder()) + + +@pytest.mark.parametrize( + "plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""] +) +def test_unsafe_plugin_names_have_no_data_folder(data_folder, plugin_name): + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_data_folder(plugin_name) + + assert not os.path.exists(data_folder) + + +def test_settings_class_knows_its_data_folder(data_folder): + folder = ExamplePluginSettings.data_folder() + + assert folder == plugin_data_folder("example-plugin") + assert os.path.isdir(folder) + assert OtherPluginSettings.data_folder() != folder + + +def test_settings_instance_uses_the_same_data_folder(config_folder, data_folder): + settings = ExamplePluginSettings.load() + + assert settings.data_folder() == ExamplePluginSettings.data_folder() + + +def test_subclass_without_its_own_name_shares_the_data_folder(data_folder): + class InheritingSettings(ExamplePluginSettings): + pass + + assert InheritingSettings.data_folder() == ExamplePluginSettings.data_folder() diff --git a/tests/common/test_plugin_updates.py b/tests/common/test_plugin_updates.py new file mode 100644 index 0000000..9c45cea --- /dev/null +++ b/tests/common/test_plugin_updates.py @@ -0,0 +1,711 @@ +import json +import shlex +from datetime import datetime, timedelta +from types import SimpleNamespace + +import pytest + +from mvt.common import logo +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP +from mvt.common.updates import ( + MVTUpdates, + PluginUpdates, + installed_plugin_distributions, +) + + +REPOSITORY_URL = "https://example.org/plugin.git" +INSTALLED_COMMIT = "a" * 40 +REMOTE_COMMIT = "b" * 40 + + +class FakeDistribution: + def __init__(self, name, version="1.0.0", direct_url=None): + self.name = name + self.version = version + self.direct_url = direct_url + + def read_text(self, file_name): + if file_name == "direct_url.json" and self.direct_url is not None: + return json.dumps(self.direct_url) + return None + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self.payload = payload or {} + + def json(self): + return self.payload + + +def _entry_point(name, distribution, value="plugin:modules"): + return SimpleNamespace(name=name, value=value, dist=distribution) + + +def _git_distribution(requested_revision=None, commit=INSTALLED_COMMIT): + vcs_info = {"vcs": "git", "commit_id": commit} + if requested_revision: + vcs_info["requested_revision"] = requested_revision + + return FakeDistribution( + "example-plugin", + direct_url={"url": REPOSITORY_URL, "vcs_info": vcs_info}, + ) + + +def _fake_git(stdout="", returncode=0, exception=None, calls=None): + def run(command, **kwargs): + if calls is not None: + calls.append((command, kwargs)) + if exception is not None: + raise exception + return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="") + + return run + + +@pytest.fixture +def data_folder(tmp_path, monkeypatch): + folder = tmp_path / "mvt-data" + monkeypatch.setattr("mvt.common.updates.MVT_DATA_FOLDER", str(folder)) + return folder + + +@pytest.fixture +def one_plugin(monkeypatch): + def install(distribution): + monkeypatch.setattr( + "mvt.common.updates.installed_plugin_distributions", + lambda: [distribution], + ) + return distribution + + return install + + +def test_installed_plugin_distributions_covers_every_plugin_group(monkeypatch): + zeta = FakeDistribution("zeta-plugin") + alpha = FakeDistribution("alpha-plugin") + neutral = FakeDistribution("neutral-plugin") + + def entry_points(*, group): + if group == MODULES_ENTRY_POINT_GROUP: + return [_entry_point("zeta", zeta), _entry_point("alpha", alpha)] + if group == IOS_CLI_PLUGIN_GROUP: + return [_entry_point("zeta-ios", zeta)] + if group == ANDROID_CLI_PLUGIN_GROUP: + return [_entry_point("alpha-android", alpha)] + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [_entry_point("shared", neutral)] + return [] + + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", entry_points + ) + + distributions = installed_plugin_distributions() + + assert [distribution.name for distribution in distributions] == [ + "alpha-plugin", + "neutral-plugin", + "zeta-plugin", + ] + + +def test_installed_plugin_distributions_skips_mvt_and_orphan_entry_points(monkeypatch): + entry_points = [ + _entry_point("builtin", FakeDistribution("mvt")), + SimpleNamespace(name="orphan", value="plugin:modules", dist=None), + _entry_point("plugin", FakeDistribution("example-plugin")), + ] + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", + lambda **kwargs: entry_points, + ) + + distributions = installed_plugin_distributions() + + assert [distribution.name for distribution in distributions] == ["example-plugin"] + + +def test_installed_plugin_distributions_survives_broken_metadata(monkeypatch, caplog): + def entry_points(*, group): + raise RuntimeError("invalid package metadata") + + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", entry_points + ) + + assert installed_plugin_distributions() == [] + assert "Unable to discover installed plugin packages" in caplog.text + + +def test_index_plugin_update_is_reported(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("example-plugin", version="1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + + findings = PluginUpdates().check() + + assert findings == [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ] + + +def test_index_plugin_queries_the_package_index_with_the_configured_timeout( + monkeypatch, data_folder, one_plugin +): + one_plugin(FakeDistribution("example-plugin")) + requests_made = [] + + def get(url, **kwargs): + requests_made.append((url, kwargs)) + return FakeResponse(payload={"info": {"version": "1.0.0"}}) + + monkeypatch.setattr("mvt.common.updates.requests.get", get) + monkeypatch.setattr("mvt.common.updates.settings.NETWORK_TIMEOUT", 3) + + PluginUpdates().check() + + assert requests_made == [ + ("https://pypi.org/pypi/example-plugin/json", {"timeout": 3}) + ] + + +def test_up_to_date_index_plugin_is_not_reported(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("example-plugin", version="1.2.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + + assert PluginUpdates().check() == [] + + +def test_unpublished_plugin_is_skipped_silently(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("private-plugin")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(status_code=404), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_following_a_branch_is_reported( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls), + ) + monkeypatch.delenv("GIT_SSH_COMMAND", raising=False) + + findings = PluginUpdates().check() + + assert findings == [ + { + "name": "example-plugin", + "installed": "aaaaaaaa", + "latest": "bbbbbbbb", + "origin": "git", + "upgrade_command": ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@main'" + ), + } + ] + command, options = calls[0] + assert command == ["git", "ls-remote", REPOSITORY_URL, "main"] + assert options["env"]["GIT_TERMINAL_PROMPT"] == "0" + # ssh asks the terminal for a passphrase or a host key unless it is told + # not to, which git itself cannot prevent. + assert options["env"]["GIT_SSH_COMMAND"] == ( + "ssh -o BatchMode=yes -o ConnectTimeout=10" + ) + + +def test_batch_mode_options_come_before_the_configured_ssh_options( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls), + ) + monkeypatch.setenv("GIT_SSH_COMMAND", "ssh -o BatchMode=no -i /home/analyst/key") + + PluginUpdates().check() + + # ssh uses the first value it is given for a keyword, so an analyst asking + # for prompts cannot bring them back, while their other options still + # apply. + assert calls[0][1]["env"]["GIT_SSH_COMMAND"] == ( + "ssh -o BatchMode=yes -o ConnectTimeout=10 -o BatchMode=no -i /home/analyst/key" + ) + + +def test_repository_plugin_without_a_revision_follows_the_default_branch( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution()) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\tHEAD\n", calls=calls), + ) + + findings = PluginUpdates().check() + + assert calls[0][0] == ["git", "ls-remote", REPOSITORY_URL, "HEAD"] + assert findings[0]["upgrade_command"] == ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}'" + ) + + +def test_hostile_revision_cannot_inject_into_the_upgrade_command( + monkeypatch, data_folder, one_plugin +): + revision = "main$(id)`id`;id" + one_plugin(_git_distribution(requested_revision=revision)) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/{revision}\n"), + ) + + upgrade_command = PluginUpdates().check()[0]["upgrade_command"] + + # Single quotes are the only quoting a shell does not expand anything in. + assert upgrade_command == ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@{revision}'" + ) + assert shlex.split(upgrade_command) == [ + "pip", + "install", + "-U", + f"example-plugin @ git+{REPOSITORY_URL}@{revision}", + ] + + +def test_repository_plugin_with_an_option_like_url_is_skipped( + monkeypatch, data_folder, one_plugin +): + one_plugin( + FakeDistribution( + "example-plugin", + direct_url={ + "url": "--upload-pack=touch /tmp/mvt", + "vcs_info": {"vcs": "git", "commit_id": INSTALLED_COMMIT}, + }, + ) + ) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_repository_plugin_with_an_option_like_revision_is_skipped( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="--upload-pack=touch /tmp/mvt")) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_repository_plugin_at_the_latest_commit_is_not_reported( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{INSTALLED_COMMIT}\trefs/heads/main\n"), + ) + + assert PluginUpdates().check() == [] + + +def test_commit_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT)) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_short_commit_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT[:10])) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_tag_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="v1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/tags/v1.0.0\n"), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_is_skipped_without_git(monkeypatch, data_folder, one_plugin): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(exception=FileNotFoundError("git")), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_is_skipped_when_git_fails( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout="", returncode=128), + ) + + assert PluginUpdates().check() == [] + + +def test_local_plugin_install_is_skipped(monkeypatch, data_folder, one_plugin): + one_plugin( + FakeDistribution( + "example-plugin", + direct_url={ + "url": "file:///home/analyst/example-plugin", + "dir_info": {"editable": True}, + }, + ) + ) + + def fail(*args, **kwargs): + raise AssertionError("a local plugin install must not be checked") + + monkeypatch.setattr("mvt.common.updates.requests.get", fail) + monkeypatch.setattr("mvt.common.updates.subprocess.run", fail) + + assert PluginUpdates().check() == [] + + +def test_check_stores_the_findings_and_the_check_timestamp( + monkeypatch, data_folder, one_plugin +): + one_plugin(FakeDistribution("example-plugin", version="1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + plugin_updates = PluginUpdates() + + findings = plugin_updates.check() + + assert json.loads((data_folder / "plugin_updates.json").read_text()) == findings + assert (data_folder / "latest_plugins_check").read_text().isdigit() + assert PluginUpdates().get_findings() == findings + + +def test_findings_are_empty_before_the_first_check(data_folder): + assert PluginUpdates().get_findings() == [] + + +def test_malformed_cached_findings_are_dropped(data_folder): + plugin_updates = PluginUpdates() + usable = { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text( + json.dumps( + [ + {"oops": 1}, + "not a finding", + {"name": "half-plugin", "installed": "1.0.0"}, + {**usable, "latest": None}, + usable, + ] + ), + encoding="utf-8", + ) + + assert plugin_updates.get_findings() == [usable] + + +def test_corrupt_cached_findings_are_ignored(data_folder): + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text("{ not json", encoding="utf-8") + + assert PluginUpdates().get_findings() == [] + + +def test_cached_findings_of_upgraded_and_removed_plugins_are_dropped( + monkeypatch, data_folder +): + findings = [ + { + "name": "upgraded-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U upgraded-plugin", + }, + { + "name": "removed-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U removed-plugin", + }, + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + }, + ] + plugin_updates = PluginUpdates() + plugin_updates.set_findings(findings) + monkeypatch.setattr( + "mvt.common.updates.installed_plugin_distributions", + lambda: [ + # The analyst upgraded this plugin since the latest check. + FakeDistribution("upgraded-plugin", version="1.2.0"), + FakeDistribution("example-plugin", version="1.0.0"), + ], + ) + + assert plugin_updates.current_findings() == [findings[2]] + + +def test_cached_findings_of_updated_repository_plugins_are_dropped(data_folder): + findings = [ + { + "name": "example-plugin", + "installed": "aaaaaaaa", + "latest": "bbbbbbbb", + "origin": "git", + "upgrade_command": "pip install -U example-plugin", + } + ] + plugin_updates = PluginUpdates() + plugin_updates.set_findings(findings) + + assert plugin_updates.current_findings([_git_distribution()]) == findings + assert ( + plugin_updates.current_findings([_git_distribution(commit=REMOTE_COMMIT)]) == [] + ) + + +def test_corrupt_check_timestamp_does_not_raise(data_folder): + plugin_updates = PluginUpdates() + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "latest_plugins_check").write_text("truncated", encoding="utf-8") + + assert plugin_updates.get_latest_check() == 0 + assert plugin_updates.should_check() == (True, 0) + + +def test_should_check_is_throttled_for_twelve_hours(data_folder): + plugin_updates = PluginUpdates() + plugin_updates.set_findings([]) + + recent = datetime.now() - timedelta(hours=4) + with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(int(recent.timestamp()))) + + should_check, hours = plugin_updates.should_check() + assert not should_check + assert hours == 8 + + old = datetime.now() - timedelta(hours=13) + with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(int(old.timestamp()))) + + assert plugin_updates.should_check() == (True, 0) + + +def test_should_check_without_a_previous_check(data_folder): + assert PluginUpdates().should_check() == (True, 0) + + +@pytest.fixture +def no_version_check(monkeypatch): + monkeypatch.setattr(MVTUpdates, "check", lambda self: "") + # Keep rich from wrapping the plugin lines while they are being asserted. + monkeypatch.setenv("COLUMNS", "200") + + +@pytest.fixture +def throttled_cache(monkeypatch, data_folder): + """Fill the findings cache and put the check inside its throttle window.""" + + def fill(findings, distributions): + PluginUpdates().set_findings(findings) + monkeypatch.setattr( + logo, "installed_plugin_distributions", lambda: distributions + ) + monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8)) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("the check must be throttled"), + ) + + return fill + + +def test_logo_prints_the_cached_plugin_updates( + capsys, no_version_check, throttled_cache +): + throttled_cache( + [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ], + [FakeDistribution("example-plugin", version="1.0.0")], + ) + + logo.check_updates(disable_indicator_check=True) + + output = capsys.readouterr().out + assert "Plugin updates available:" in output + assert "example-plugin 1.0.0 → 1.2.0 (pip install -U example-plugin)" in output + + +def test_logo_does_not_print_a_cached_update_of_an_upgraded_plugin( + capsys, no_version_check, throttled_cache +): + throttled_cache( + [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ], + # The analyst already upgraded the plugin the cached finding is about. + [FakeDistribution("example-plugin", version="1.2.0")], + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_prints_nothing_when_throttled_without_findings( + capsys, no_version_check, throttled_cache +): + throttled_cache([], [FakeDistribution("example-plugin")]) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_survives_a_corrupt_plugin_cache( + monkeypatch, capsys, data_folder, no_version_check +): + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text( + json.dumps([{"oops": 1}]), encoding="utf-8" + ) + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: [FakeDistribution("example-plugin")], + ) + monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8)) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_without_plugins( + monkeypatch, capsys, no_version_check +): + monkeypatch.setattr(logo, "installed_plugin_distributions", list) + monkeypatch.setattr( + PluginUpdates, + "should_check", + lambda self: pytest.fail("plugins must not be checked without plugins"), + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_without_network_access( + monkeypatch, capsys, no_version_check +): + monkeypatch.setattr("mvt.common.logo.settings.NETWORK_ACCESS_ALLOWED", False) + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: pytest.fail("plugins must not be listed without network access"), + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_when_update_checks_are_disabled( + monkeypatch, capsys +): + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: pytest.fail("plugins must not be checked with --disable-update-check"), + ) + + logo.check_updates(disable_version_check=True, disable_indicator_check=True) + + assert capsys.readouterr().out == "" diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index 4dbe5c0..6bef147 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -8,6 +8,7 @@ import logging import os from datetime import datetime +from mvt.common.log import MVTLogHandler from mvt.common.utils import ( CustomJSONEncoder, convert_datetime_to_iso, @@ -16,6 +17,8 @@ from mvt.common.utils import ( convert_unix_to_utc_datetime, generate_hashes_from_path, get_sha256_from_file_path, + init_logging, + set_verbose_logging, ) from ..utils import get_artifact_folder @@ -103,3 +106,46 @@ class TestCustomJSONEncoder: json.dumps({"name": "家".encode()}, cls=CustomJSONEncoder) == '{"name": "\\u5bb6"}' ) + + +class TestInitLogging: + def test__init_logging_is_idempotent(self): + # Loaded module packages may import an MVT CLI module, which calls + # init_logging() again at import time. A second call must not add + # a duplicate console handler. + log = logging.getLogger("mvt") + init_logging() + handler_count = sum( + isinstance(handler, MVTLogHandler) for handler in log.handlers + ) + init_logging() + assert ( + sum(isinstance(handler, MVTLogHandler) for handler in log.handlers) + == handler_count + ) + + def test_verbose_logging_finds_the_console_handler_among_others(self): + # Something else may have attached a handler to the "mvt" logger + # before MVT did, so the console handler is not always the first. + log = logging.getLogger("mvt") + init_logging() + foreign_handler = logging.NullHandler() + foreign_handler.setLevel(logging.CRITICAL) + log.handlers.insert(0, foreign_handler) + + try: + set_verbose_logging(True) + console_handlers = [ + handler + for handler in log.handlers + if isinstance(handler, MVTLogHandler) + ] + assert console_handlers + assert all(handler.level == logging.DEBUG for handler in console_handlers) + assert foreign_handler.level == logging.CRITICAL + + set_verbose_logging(False) + assert all(handler.level == logging.INFO for handler in console_handlers) + assert foreign_handler.level == logging.CRITICAL + finally: + log.handlers.remove(foreign_handler) diff --git a/tests/conftest.py b/tests/conftest.py index c89f629..06a890a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,11 @@ import os import pytest +from mvt.common.cli_plugins import ( + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, +) from mvt.common.indicators import Indicators from .artifacts.generate_stix import generate_test_stix_file @@ -58,3 +63,33 @@ def indicators_factory(indicator_file): return ind return f + + +@pytest.fixture() +def restore_cli_commands(monkeypatch): + """Keep the external commands a test registers out of the next test. + + Each CLI group is a module-level object shared by every test, so a test + registering plugin or environment commands on one has to put it back. The + groups are imported here rather than at the top of the file, so that + collecting the tests does not import three CLIs for the sake of one + fixture. + """ + from mvt.android.cli import cli as android_cli + from mvt.cli import cli as neutral_cli + from mvt.ios.cli import cli as ios_cli + + groups = (neutral_cli, ios_cli, android_cli) + for variable in ( + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ): + monkeypatch.delenv(variable, raising=False) + originals = [dict(group.commands) for group in groups] + yield + for group, commands in zip(groups, originals): + group.commands.clear() + group.commands.update(commands) + if hasattr(group, "_mvt_external_command_sources"): + delattr(group, "_mvt_external_command_sources") diff --git a/tests/ios_backup/test_interactionc.py b/tests/ios_backup/test_interactionc.py new file mode 100644 index 0000000..eec2656 --- /dev/null +++ b/tests/ios_backup/test_interactionc.py @@ -0,0 +1,105 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.common.module import run_module +from mvt.ios.modules.mixed.interactionc import InteractionC +from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts + +from ..utils import get_ios_backup_folder + + +class TestInteractionCModule: + def test_extraction_with_whatsapp_contacts(self): + contacts = WhatsappContacts(target_path=get_ios_backup_folder()) + run_module(contacts) + + m = InteractionC(target_path=get_ios_backup_folder()) + m.dependency_modules = {WhatsappContacts: contacts} + run_module(m) + + assert len(m.results) == 3 + + incoming = next( + r for r in m.results if r["sender_identifier"] == "100000000000001@lid" + ) + assert incoming["direction"] == "INCOMING" + assert incoming["sender_resolved_phone_number"] == "+14155550100" + assert incoming["sender_resolved_name"] == "Alice Example" + + outgoing = next( + r for r in m.results if r["direction"] == "OUTGOING" + ) + assert outgoing["recipient_identifier"] == "+14155550100" + assert outgoing["recipient_resolved_name"] == "Alice Example" + assert outgoing["domain_resolved_phone_number"] == "+14155550100" + assert outgoing["domain_resolved_name"] == "Alice Example" + + sms = next( + r for r in m.results if r["bundle_id"] == "com.apple.MobileSMS" + ) + assert sms.get("sender_resolved_name") is None + assert sms["sender_display_name"] == "Bob Example" + + events = [entry["data"] for entry in m.timeline] + assert ( + "[net.whatsapp.WhatsApp] INCOMING from " + "Alice Example (+14155550100) to local user" in events + ) + assert ( + "[net.whatsapp.WhatsApp] OUTGOING from local user to " + "Alice Example (+14155550100)" in events + ) + assert ( + "[com.apple.MobileSMS] INCOMING from " + "Bob Example (+14155550101) to local user" in events + ) + + # The creation date is only serialized when it diverges from the + # start date; the SMS record was created 90 days after the event. + creation_events = [ + entry + for entry in m.timeline + if entry["event"] == "interactions_creation_date" + ] + assert len(creation_events) == 1 + assert creation_events[0]["timestamp"] == "2025-12-09 12:26:40.000000" + assert creation_events[0]["data"] == ( + "Interaction record created 90 days after the event: " + "[com.apple.MobileSMS] INCOMING from " + "Bob Example (+14155550101) to local user" + ) + + # Per-contact aggregate dates use contact-centric data strings. + first_seen = [ + entry + for entry in m.timeline + if entry["event"] == "first_incoming_sender_date" + ] + assert len(first_seen) == 1 + assert first_seen[0]["timestamp"] == "2025-09-03 13:46:40.000000" + assert first_seen[0]["data"] == ( + "First incoming interaction from Bob Example (+14155550101)" + ) + assert ( + "Last incoming interaction from Bob Example (+14155550101)" + in events + ) + + def test_extraction_without_whatsapp_contacts(self): + # Without the WhatsappContacts dependency the module still runs, and + # unresolvable LIDs are shown as-is. + m = InteractionC(target_path=get_ios_backup_folder()) + run_module(m) + + assert len(m.results) == 3 + events = [entry["data"] for entry in m.timeline] + assert ( + "[net.whatsapp.WhatsApp] INCOMING from " + "100000000000001@lid to local user" in events + ) + assert ( + "[net.whatsapp.WhatsApp] OUTGOING from local user to " + "+14155550100" in events + ) diff --git a/tests/ios_backup/test_whatsapp.py b/tests/ios_backup/test_whatsapp.py index 864fb84..3bc920d 100644 --- a/tests/ios_backup/test_whatsapp.py +++ b/tests/ios_backup/test_whatsapp.py @@ -6,8 +6,79 @@ import logging from mvt.common.indicators import Indicators +from mvt.common.module import run_module from mvt.ios.modules.mixed.whatsapp import Whatsapp +from ..utils import get_ios_backup_folder + + +def test_extraction(): + m = Whatsapp(target_path=get_ios_backup_folder()) + run_module(m) + + messages = [r for r in m.results if "ZTEXT" in r] + sessions = [r for r in m.results if r.get("record_type") == "chat_session"] + pairs = [ + r for r in m.results + if r.get("record_type") == "lid_phone_number_pair" + ] + assert len(messages) == 3 + assert len(sessions) == 2 + assert len(pairs) == 1 + + assert pairs[0]["lid"] == "100000000000001" + assert pairs[0]["phone_number"] == "14155550100" + assert pairs[0]["pair_timestamp"] == "2025-08-25 07:33:20.000000" + + linked = next(r for r in messages if r.get("links")) + assert linked["links"] == ["https://example.org/news"] + + alice = next(s for s in sessions if s["partner_name"] == "Alice Example") + assert alice["contact_jid"] == "100000000000001@lid" + assert alice["partner_resolved_phone_number"] == "+14155550100" + assert alice["first_stored_message_date"] == "2025-08-27 15:06:40.000000" + assert alice["last_message_date"] == "2025-08-28 18:53:20.000000" + assert alice["group_creation_date"] is None + assert alice["stored_message_count"] == 2 + + group = next(s for s in sessions if s["partner_name"] == "Example Group") + assert group["group_creation_date"] == "2025-08-21 20:13:20.000000" + assert group["first_stored_message_date"] == "2025-08-29 22:40:00.000000" + # The last stored message predates the session's own last-message date: + # the newest message in this chat was deleted. + assert group["last_stored_message_date"] == "2025-08-29 22:40:00.000000" + assert group["last_message_date"] == "2025-08-31 02:26:40.000000" + + # 3 message events, first/last per chat, the group creation and the + # LID-phone number pair. + assert len(m.timeline) == 9 + events = { + (entry["event"], entry["timestamp"]): entry["data"] + for entry in m.timeline + } + # Alice's session is keyed by LID but labelled with the phone number + # resolved through LID.sqlite. + assert events[("chat_first_message", "2025-08-27 15:06:40.000000")] == ( + "First stored message in WhatsApp chat with " + "'Alice Example' (+14155550100)" + ) + assert events[("chat_last_message", "2025-08-28 18:53:20.000000")] == ( + "Last message in WhatsApp chat with " + "'Alice Example' (+14155550100)" + ) + assert events[("lid_pair_recorded", "2025-08-25 07:33:20.000000")] == ( + "WhatsApp associated LID 100000000000001 with " + "phone number 14155550100" + ) + assert events[("group_created", "2025-08-21 20:13:20.000000")] == ( + "WhatsApp group chat 'Example Group' " + "(120000000000000001@g.us) was created" + ) + assert ("chat_first_message", "2025-08-29 22:40:00.000000") in events + assert ("chat_last_message", "2025-08-31 02:26:40.000000") in events + + assert len(m.alertstore.alerts) == 0 + def test_collect_url_results_includes_expansion(): module = Whatsapp( diff --git a/tests/ios_backup/test_whatsapp_contacts.py b/tests/ios_backup/test_whatsapp_contacts.py new file mode 100644 index 0000000..a91b8f1 --- /dev/null +++ b/tests/ios_backup/test_whatsapp_contacts.py @@ -0,0 +1,83 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.common.module import run_module +from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts + +from ..utils import get_ios_backup_folder + + +class TestWhatsappContactsModule: + def test_extraction(self): + m = WhatsappContacts(target_path=get_ios_backup_folder()) + run_module(m) + assert len(m.results) == 2 + + alice = next(r for r in m.results if r["given_name"] == "Alice") + assert alice["full_name"] == "Alice Example" + assert alice["phone_number"] == "+14155550100" + assert alice["whatsapp_id"] == "14155550100@s.whatsapp.net" + assert alice["lid"] == "100000000000001@lid" + assert alice["user_name"] == "alice.example" + assert alice["disappearing_mode_duration"] == 86400.0 + assert alice["disappearing_mode_is_on"] is True + assert alice["disappearing_mode_label"] == "24 hours" + assert alice["disappearing_mode_timestamp"] == "2025-07-23 21:46:40.000000" + assert alice["about_timestamp"] == "2025-07-12 08:00:00.000000" + assert alice["about_expiration_timestamp"] == "2025-08-16 01:20:00.000000" + assert alice["last_updated"] == "2025-08-04 11:33:20.000000" + + bob = next(r for r in m.results if r["given_name"] == "Bob") + assert bob["lid"] is None + assert bob["disappearing_mode_duration"] is None + assert bob["disappearing_mode_is_on"] is False + assert bob["disappearing_mode_label"] == "off" + assert bob["disappearing_mode_timestamp"] is None + + # Alice: disappearing_mode_set, about_changed, about_expiration and + # contact_last_updated. Bob: contact_last_updated only. + assert len(m.timeline) == 5 + + events = { + (entry["event"], entry["timestamp"]): entry["data"] + for entry in m.timeline + } + assert ( + "24 hours" + in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")] + ) + assert ( + "14155550100@s.whatsapp.net (Alice Example)" + in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")] + ) + assert ( + 'changed to "Hey there! I am using WhatsApp."' + in events[("about_changed", "2025-07-12 08:00:00.000000")] + ) + assert ( + "scheduled to expire" + in events[("about_expiration", "2025-08-16 01:20:00.000000")] + ) + + updated = [ + entry["data"] + for entry in m.timeline + if entry["event"] == "contact_last_updated" + ] + assert len(updated) == 2 + assert all( + entry["timestamp"] == "2025-08-04 11:33:20.000000" + for entry in m.timeline + if entry["event"] == "contact_last_updated" + ) + assert any("14155550101@s.whatsapp.net (Bob Example)" in d for d in updated) + + assert len(m.alertstore.alerts) == 0 + + def test_missing_database(self, tmp_path): + m = WhatsappContacts(target_path=str(tmp_path)) + run_module(m) + assert m.results == [] + assert len(m.alertstore.alerts) == 0 diff --git a/tests/ios_fs/test_filesystem.py b/tests/ios_fs/test_filesystem.py index 9fa664f..636c004 100644 --- a/tests/ios_fs/test_filesystem.py +++ b/tests/ios_fs/test_filesystem.py @@ -15,8 +15,8 @@ class TestFilesystem: def test_filesystem(self): m = Filesystem(target_path=get_ios_backup_folder()) run_module(m) - assert len(m.results) == 15 - assert len(m.timeline) == 15 + assert len(m.results) == 23 + assert len(m.timeline) == 23 assert len(m.alertstore.alerts) == 0 def test_detection(self, indicator_file): @@ -29,6 +29,6 @@ class TestFilesystem: ) m.indicators = ind run_module(m) - assert len(m.results) == 15 - assert len(m.timeline) == 15 + assert len(m.results) == 23 + assert len(m.timeline) == 23 assert len(m.alertstore.alerts) == 1 diff --git a/tests/plugin_fixtures.py b/tests/plugin_fixtures.py new file mode 100644 index 0000000..ef989dc --- /dev/null +++ b/tests/plugin_fixtures.py @@ -0,0 +1,86 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Helpers building throwaway plugin distributions for the tests. + +Some plugin behaviour only shows up in a fresh interpreter: what an import +executes, and what a plugin sees when MVT is imported before or after it. +These helpers write an importable distribution with a real entry point and +run a script against it in a subprocess, with a temporary home so that the +subprocess cannot touch the configuration of whoever runs the tests. +""" + +import os +import subprocess +import sys +from pathlib import Path +from typing import Optional + +FIXTURE_COMMAND_NAME = "fixture-plugin" +FIXTURE_MODULE_NAME = "fixture_cli_plugin" +FIXTURE_DISTRIBUTION_NAME = "fixture-cli-plugin" + + +def write_cli_plugin_distribution( + site_path: Path, + entry_point_group: str, + module_source: str, +) -> Path: + """Write a distribution registering a CLI plugin entry point. + + :param site_path: Folder to write the distribution into, to be added to + the import path of the interpreter loading it. + :param entry_point_group: Entry-point group to register the command in. + :param module_source: Source of the plugin module, which must define a + Click command named `cli`. + :returns: The folder the distribution was written to. + """ + site_path.mkdir(parents=True, exist_ok=True) + (site_path / f"{FIXTURE_MODULE_NAME}.py").write_text( + module_source, encoding="utf-8" + ) + + dist_info = ( + site_path / f"{FIXTURE_DISTRIBUTION_NAME.replace('-', '_')}-1.0.dist-info" + ) + dist_info.mkdir(exist_ok=True) + (dist_info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {FIXTURE_DISTRIBUTION_NAME}\nVersion: 1.0\n", + encoding="utf-8", + ) + (dist_info / "entry_points.txt").write_text( + f"[{entry_point_group}]\n{FIXTURE_COMMAND_NAME} = {FIXTURE_MODULE_NAME}:cli\n", + encoding="utf-8", + ) + return site_path + + +def run_isolated_python( + script: str, + home: Path, + site_path: Optional[Path] = None, + **environment: str, +) -> subprocess.CompletedProcess: + """Run a script in a fresh interpreter with its own configuration folder. + + Importing MVT writes its configuration file, so the subprocess gets a + temporary home and no MVT environment variables from the test session. + """ + isolated_environment = { + key: value for key, value in os.environ.items() if not key.startswith("MVT_") + } + isolated_environment["HOME"] = str(home) + isolated_environment["XDG_CONFIG_HOME"] = str(home / "config") + isolated_environment["XDG_DATA_HOME"] = str(home / "data") + if site_path is not None: + isolated_environment["PYTHONPATH"] = str(site_path) + isolated_environment.update(environment) + + return subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=isolated_environment, + ) diff --git a/tests/test_check_android_androidqf.py b/tests/test_check_android_androidqf.py index 2253a50..4e6ef98 100644 --- a/tests/test_check_android_androidqf.py +++ b/tests/test_check_android_androidqf.py @@ -155,7 +155,7 @@ class TestCheckAndroidqfCommand: result = runner.invoke(check_androidqf, [str(path)]) assert result.exit_code == 0 - assert "Skipping backup modules as backup.ab is malformed" in caplog.text + assert "Skipping backup modules: Invalid backup format" in caplog.text assert not any( record.levelname in {"CRITICAL", "FATAL"} for record in caplog.records ) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6495243 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,48 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from click.testing import CliRunner + +from mvt.cli import cli +from mvt.common.updates import IndicatorsUpdates +from mvt.common.version import MVT_VERSION + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + + +class TestMvtCommand: + def test_running_mvt_alone_shows_the_logo_and_the_commands(self): + result = CliRunner().invoke(cli, OFFLINE) + + assert result.exit_code == 0 + logo_at = result.output.index("Mobile Verification Toolkit") + usage_at = result.output.index("Usage:") + assert logo_at < usage_at + assert "mvt-ios" in result.output and "mvt-android" in result.output + + def test_help_reminds_where_the_analysis_runs(self): + result = CliRunner().invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "mvt-ios" in result.output + assert "mvt-android" in result.output + + def test_version_prints_the_installed_version(self): + result = CliRunner().invoke(cli, [*OFFLINE, "version"]) + + assert result.exit_code == 0 + assert f"Version: {MVT_VERSION}" in result.output + + def test_download_iocs_updates_the_indicators(self, monkeypatch): + updates = [] + monkeypatch.setattr( + IndicatorsUpdates, "update", lambda self: updates.append(self) + ) + + result = CliRunner().invoke(cli, [*OFFLINE, "download-iocs"]) + + assert result.exit_code == 0 + assert len(updates) == 1 diff --git a/tests/test_cli_entry_points.py b/tests/test_cli_entry_points.py new file mode 100644 index 0000000..1b51413 --- /dev/null +++ b/tests/test_cli_entry_points.py @@ -0,0 +1,280 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import sys +from types import SimpleNamespace + +import click +import pytest + +import mvt.android +import mvt.cli +import mvt.ios +from mvt.android.cli import cli as android_cli +from mvt.android.cli import main as android_main +from mvt.cli import cli as mvt_cli +from mvt.cli import main as mvt_main +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.ios.cli import cli as ios_cli +from mvt.ios.cli import main as ios_main + +from .plugin_fixtures import ( + FIXTURE_COMMAND_NAME, + run_isolated_python, + write_cli_plugin_distribution, +) + +MARKER_PLUGIN_TEMPLATE = """ +import os + +import click + +# Touched when this module is imported, so a test can tell whether loading MVT +# executed the plugin. +open(os.environ["FIXTURE_PLUGIN_MARKER"], "a").close() + + +@click.command() +def cli(): + click.echo("fixture plugin ran") +""" + +PROGRAMS = { + "mvt": (mvt.cli, mvt_cli, NEUTRAL_CLI_PLUGIN_GROUP, MVT_CUSTOM_COMMANDS_ENV), + "mvt-ios": (mvt.ios, ios_cli, IOS_CLI_PLUGIN_GROUP, MVT_IOS_CUSTOM_COMMANDS_ENV), + "mvt-android": ( + mvt.android, + android_cli, + ANDROID_CLI_PLUGIN_GROUP, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ), +} + +CASE_SUMMARY_COMMAND = """ +import click + + +@click.command("case-summary") +def cli(): + click.echo("case summary ran") +""" + +# The entry-point group of another program, for each program: no group may add +# its commands to a CLI other than its own. +OTHER_PROGRAMS_GROUP = { + "mvt": IOS_CLI_PLUGIN_GROUP, + "mvt-ios": NEUTRAL_CLI_PLUGIN_GROUP, + "mvt-android": NEUTRAL_CLI_PLUGIN_GROUP, +} + + +def _install_fixture_entry_point(monkeypatch, entry_point_group, command): + def entry_points(*, group): + if group != entry_point_group: + return [] + return [ + SimpleNamespace( + name=FIXTURE_COMMAND_NAME, + value="fixture_cli_plugin:cli", + load=lambda: command, + dist=SimpleNamespace( + metadata={"Name": "fixture-cli-plugin"}, version="1.0" + ), + ) + ] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", entry_points + ) + + +def _offline_argv(program, *arguments): + """Build an argument list which keeps the CLI from checking for updates.""" + return [ + program, + "--disable-update-check", + "--disable-indicator-update-check", + *arguments, + ] + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_registers_installed_plugins_before_running_the_cli( + program, monkeypatch, capsys, restore_cli_commands +): + package, group, entry_point_group, _ = PROGRAMS[program] + + @click.command() + def fixture_command(): + click.echo("fixture plugin ran") + + _install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command) + monkeypatch.setattr(sys, "argv", _offline_argv(program, FIXTURE_COMMAND_NAME)) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "fixture plugin ran" in capsys.readouterr().out + assert FIXTURE_COMMAND_NAME in group.commands + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_completes_plugin_command_names( + program, monkeypatch, capsys, restore_cli_commands +): + package, _, entry_point_group, _ = PROGRAMS[program] + + @click.command() + def fixture_command(): + pass + + _install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command) + complete_variable = f"_{program.upper().replace('-', '_')}_COMPLETE" + monkeypatch.setenv(complete_variable, "bash_complete") + monkeypatch.setenv("COMP_WORDS", f"{program} fixture") + monkeypatch.setenv("COMP_CWORD", "1") + monkeypatch.setattr(sys, "argv", [program]) + + with pytest.raises(SystemExit): + package.main() + + assert f"plain,{FIXTURE_COMMAND_NAME}" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_still_loads_commands_from_a_file( + program, monkeypatch, capsys, tmp_path, restore_cli_commands +): + package, _, entry_point_group, _ = PROGRAMS[program] + command_path = tmp_path / "case_summary.py" + command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8") + _install_fixture_entry_point( + monkeypatch, entry_point_group, click.Command("unused") + ) + monkeypatch.setattr( + sys, + "argv", + _offline_argv(program, "--load-command", str(command_path), "case-summary"), + ) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "case summary ran" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_loads_commands_from_the_environment_variable( + program, monkeypatch, capsys, tmp_path, restore_cli_commands +): + # Each CLI reads its own variable, so a main() reading another CLI's would + # go unnoticed without this. + package, _, _, environment_variable = PROGRAMS[program] + command_path = tmp_path / "case_summary.py" + command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8") + monkeypatch.setenv(environment_variable, str(command_path)) + monkeypatch.setattr(sys, "argv", _offline_argv(program, "case-summary")) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "case summary ran" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_ignores_the_entry_point_groups_of_the_other_programs( + program, monkeypatch, capsys, restore_cli_commands +): + package, group, _, _ = PROGRAMS[program] + _install_fixture_entry_point( + monkeypatch, + OTHER_PROGRAMS_GROUP[program], + click.Command(FIXTURE_COMMAND_NAME), + ) + monkeypatch.setattr(sys, "argv", _offline_argv(program, "--help")) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert FIXTURE_COMMAND_NAME not in group.commands + assert FIXTURE_COMMAND_NAME not in capsys.readouterr().out + + +def test_the_console_script_targets_are_importable(): + # [project.scripts] points at these, so they must stay where they are. + assert mvt.cli.main is mvt_main + assert mvt.ios.main is ios_main + assert mvt.android.main is android_main + + +def test_importing_mvt_does_not_import_a_cli(tmp_path): + # The mvt package deliberately re-exports nothing of mvt.cli, so that + # importing MVT stays cheap and free of side effects. + result = run_isolated_python( + "import sys\n" + "import mvt\n" + "print('imported a cli' if 'mvt.cli' in sys.modules else 'imported mvt')\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "imported mvt" + + +def test_importing_mvt_does_not_run_installed_plugins(tmp_path): + site_path = write_cli_plugin_distribution( + tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE + ) + marker = tmp_path / "plugin-imported" + + result = run_isolated_python( + "import mvt.ios.cli\nimport mvt.android.cli\nprint('imported')", + home=tmp_path / "home", + site_path=site_path, + FIXTURE_PLUGIN_MARKER=str(marker), + ) + + assert result.returncode == 0, result.stderr + assert "imported" in result.stdout + assert not marker.exists() + + +def test_registering_the_plugins_runs_the_entry_point(tmp_path): + site_path = write_cli_plugin_distribution( + tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE + ) + marker = tmp_path / "plugin-imported" + + result = run_isolated_python( + "import click\n" + "from mvt.common.cli_plugins import (\n" + " IOS_CLI_PLUGIN_GROUP,\n" + " BrokenPluginCommand,\n" + " register_installed_cli_commands,\n" + ")\n" + "group = click.Group()\n" + "register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)\n" + f"command = group.commands[{FIXTURE_COMMAND_NAME!r}]\n" + "assert not isinstance(command, BrokenPluginCommand), command.help\n" + "print('registered')\n", + home=tmp_path / "home", + site_path=site_path, + FIXTURE_PLUGIN_MARKER=str(marker), + ) + + assert result.returncode == 0, result.stderr + assert "registered" in result.stdout + assert marker.exists() diff --git a/tests/test_cli_startup.py b/tests/test_cli_startup.py new file mode 100644 index 0000000..e009256 --- /dev/null +++ b/tests/test_cli_startup.py @@ -0,0 +1,28 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import pytest + +from .plugin_fixtures import run_isolated_python + +# Importing a platform CLI must only build its command tree: the console +# scripts import it before Click can answer a shell completion request, which +# the completion scripts make on every keystroke. Every command imports what +# it runs when it is invoked. Each of these costs tens of milliseconds to +# import and is the sign that a command implementation is imported too early. +HEAVY_MODULES = ("pydantic", "requests", "Crypto", "mvt.common.module") + + +@pytest.mark.parametrize("cli_module", ("mvt.ios.cli", "mvt.android.cli")) +def test_importing_a_cli_does_not_import_the_module_machinery(cli_module, tmp_path): + result = run_isolated_python( + "import sys\n" + f"import {cli_module}\n" + f"print(','.join(name for name in {HEAVY_MODULES!r} if name in sys.modules))\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "", f"{cli_module} imported {result.stdout.strip()}" diff --git a/tests/test_cli_verbose.py b/tests/test_cli_verbose.py new file mode 100644 index 0000000..d8826ff --- /dev/null +++ b/tests/test_cli_verbose.py @@ -0,0 +1,107 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli +from mvt.common.log import MVTLogHandler +from mvt.common.utils import set_verbose_logging +from mvt.ios.cli import cli as ios_cli + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + +PROGRAMS = {"mvt": mvt_cli, "mvt-ios": ios_cli, "mvt-android": android_cli} + + +@pytest.fixture(autouse=True) +def _reset_console_level(): + """Leave the console handler at its default level after every test.""" + yield + set_verbose_logging(False) + + +def _console_level(): + """Return the level of MVT's own console log handler.""" + for handler in logging.getLogger("mvt").handlers: + if isinstance(handler, MVTLogHandler): + return handler.level + raise AssertionError("MVT has no console log handler") + + +class TestVerboseOnTheCommands: + @pytest.mark.parametrize("program", sorted(PROGRAMS)) + def test_verbose_before_the_command_name_turns_on_debug(self, program): + cli = PROGRAMS[program] + + result = CliRunner().invoke(cli, [*OFFLINE, "--verbose", "version"]) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + @pytest.mark.parametrize("program", sorted(PROGRAMS)) + def test_a_run_without_verbose_goes_back_to_info(self, program): + cli = PROGRAMS[program] + CliRunner().invoke(cli, [*OFFLINE, "--verbose", "version"]) + + result = CliRunner().invoke(cli, [*OFFLINE, "version"]) + + assert result.exit_code == 0 + assert _console_level() == logging.INFO + + def test_mvt_verbose_without_a_command_prints_the_help(self): + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "--verbose"]) + + assert result.exit_code == 0 + assert "Usage:" in result.output + assert _console_level() == logging.DEBUG + + +class TestVerboseOnTheCheckCommands: + def test_ios_command_default_does_not_undo_the_cli_choice(self, tmp_path): + result = CliRunner().invoke( + ios_cli, + [*OFFLINE, "--verbose", "check-backup", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_ios_verbose_after_the_command_name_still_works(self, tmp_path): + result = CliRunner().invoke( + ios_cli, + [*OFFLINE, "check-backup", "--verbose", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_android_command_default_does_not_undo_the_cli_choice(self, tmp_path): + result = CliRunner().invoke( + android_cli, + [*OFFLINE, "--verbose", "check-bugreport", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_android_verbose_after_the_command_name_still_works(self, tmp_path): + result = CliRunner().invoke( + android_cli, + [*OFFLINE, "check-bugreport", "--verbose", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_the_command_option_says_it_is_kept_for_compatibility(self): + result = CliRunner().invoke(ios_cli, [*OFFLINE, "check-backup", "--help"]) + + assert result.exit_code == 0 + assert "kept for compatibility" in result.output diff --git a/tests/test_completion.py b/tests/test_completion.py index 48c0177..08ad3ce 100644 --- a/tests/test_completion.py +++ b/tests/test_completion.py @@ -6,56 +6,60 @@ from click.testing import CliRunner from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli from mvt.ios.cli import cli as ios_cli class TestCompletionCommand: def test_completion_prints_instructions_by_default(self): runner = CliRunner() - result = runner.invoke(ios_cli, ["completion"]) + result = runner.invoke(mvt_cli, ["completion"]) assert result.exit_code == 0 - assert "Shell completion for mvt-ios" in result.output - assert "mvt-ios completion bash > ~/.mvt-ios-complete.bash" in result.output + assert "Shell completion for mvt, mvt-ios and mvt-android" in result.output + assert "mvt completion bash > ~/.mvt-complete.bash" in result.output assert "Mobile Verification Toolkit" not in result.output - def test_completion_prints_bash_script(self): + def test_completion_bash_script_covers_every_cli(self): runner = CliRunner() - result = runner.invoke(ios_cli, ["completion", "bash"]) + result = runner.invoke(mvt_cli, ["completion", "bash"]) assert result.exit_code == 0 + assert "_MVT_COMPLETE=bash_complete" in result.output assert "_MVT_IOS_COMPLETE=bash_complete" in result.output + assert "_MVT_ANDROID_COMPLETE=bash_complete" in result.output assert "complete -o nosort" in result.output - assert "mvt-ios" in result.output assert "Mobile Verification Toolkit" not in result.output - def test_completion_prints_fish_script(self): + def test_completion_fish_script_covers_every_cli(self): runner = CliRunner() - result = runner.invoke(android_cli, ["completion", "fish"]) + result = runner.invoke(mvt_cli, ["completion", "fish"]) assert result.exit_code == 0 - assert "_MVT_ANDROID_COMPLETE=fish_complete" in result.output + assert "complete --no-files --command mvt-ios" in result.output assert "complete --no-files --command mvt-android" in result.output + assert "complete --no-files --command mvt " in result.output assert "Mobile Verification Toolkit" not in result.output def test_completion_install_updates_bashrc_once(self, tmp_path, monkeypatch): monkeypatch.setenv("HOME", str(tmp_path)) runner = CliRunner() - result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "bash", "--install"]) assert result.exit_code == 0 - script_path = tmp_path / ".mvt-ios-complete.bash" + script_path = tmp_path / ".mvt-complete.bash" bashrc_path = tmp_path / ".bashrc" assert script_path.exists() - assert "_MVT_IOS_COMPLETE=bash_complete" in script_path.read_text( - encoding="utf-8" - ) + script = script_path.read_text(encoding="utf-8") + assert "_MVT_COMPLETE=bash_complete" in script + assert "_MVT_IOS_COMPLETE=bash_complete" in script + assert "_MVT_ANDROID_COMPLETE=bash_complete" in script bashrc = bashrc_path.read_text(encoding="utf-8") assert "[ -f" in bashrc - assert ".mvt-ios-complete.bash" in bashrc + assert ".mvt-complete.bash" in bashrc - result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "bash", "--install"]) assert result.exit_code == 0 assert bashrc_path.read_text(encoding="utf-8") == bashrc @@ -65,14 +69,30 @@ class TestCompletionCommand: monkeypatch.setenv("HOME", str(tmp_path)) runner = CliRunner() - result = runner.invoke(android_cli, ["completion", "fish", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "fish", "--install"]) assert result.exit_code == 0 - script_path = ( - tmp_path / ".config" / "fish" / "completions" / "mvt-android.fish" - ) + script_path = tmp_path / ".config" / "fish" / "conf.d" / "mvt-completion.fish" assert script_path.exists() - assert "_MVT_ANDROID_COMPLETE=fish_complete" in script_path.read_text( - encoding="utf-8" - ) + script = script_path.read_text(encoding="utf-8") + assert "_MVT_COMPLETE=fish_complete" in script + assert "_MVT_IOS_COMPLETE=fish_complete" in script + assert "_MVT_ANDROID_COMPLETE=fish_complete" in script assert not (tmp_path / ".fishrc").exists() + assert not (tmp_path / ".bashrc").exists() + assert not (tmp_path / ".zshrc").exists() + + def test_completion_install_without_shell_is_a_usage_error(self): + runner = CliRunner() + result = runner.invoke(mvt_cli, ["completion", "--install"]) + + assert result.exit_code == 2 + assert "A shell is required when using --install." in result.output + + def test_completion_is_not_a_command_of_the_platform_clis(self): + runner = CliRunner() + + assert "completion" not in ios_cli.commands + assert "completion" not in android_cli.commands + assert runner.invoke(ios_cli, ["completion"]).exit_code == 2 + assert runner.invoke(android_cli, ["completion"]).exit_code == 2 diff --git a/tests/test_custom_modules.py b/tests/test_custom_modules.py index a00faea..43ab621 100644 --- a/tests/test_custom_modules.py +++ b/tests/test_custom_modules.py @@ -1,3 +1,7 @@ +import hashlib +import importlib.metadata +import json + from click.testing import CliRunner from mvt.android.cli import check_bugreport @@ -5,7 +9,9 @@ from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF from mvt.android.cmd_check_backup import CmdAndroidCheckBackup from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs +from mvt.common import module_loader from mvt.common.module import MVTModule +from mvt.common.version import MVT_VERSION from mvt.ios.cli import check_backup, check_fs @@ -106,6 +112,150 @@ def test_custom_modules_load_from_environment_without_cli_flag(tmp_path, monkeyp assert "EnvBugreportModule" in result.output +class InstalledPackageModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + + +def get_installed_package_modules(): + return [InstalledPackageModule] + + +def _fake_entry_points(monkeypatch, value, name="test-modules"): + entry_point = importlib.metadata.EntryPoint( + name=name, value=value, group=module_loader.MODULES_ENTRY_POINT_GROUP + ) + + def fake_entry_points(*, group): + assert group == module_loader.MODULES_ENTRY_POINT_GROUP + return [entry_point] + + monkeypatch.setattr( + module_loader.importlib.metadata, "entry_points", fake_entry_points + ) + + +def test_installed_module_package_loads_from_entry_point(monkeypatch): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + + modules = module_loader.load_custom_modules() + + assert modules == [InstalledPackageModule] + + +def test_broken_module_entry_point_is_skipped(monkeypatch, caplog): + _fake_entry_points(monkeypatch, "nonexistent_module_xyz:get_modules") + + with caplog.at_level("WARNING"): + modules = module_loader.load_custom_modules() + + assert modules == [] + assert "Unable to load modules from entry point" in caplog.text + + +def test_entry_point_module_deduplicated_against_paths(monkeypatch, tmp_path): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + module_path = _write_custom_module( + tmp_path / "custom.py", + "PathLoadedModule", + (("ios", "check-backup"),), + ) + + modules = module_loader.load_custom_modules([str(module_path)]) + + assert [module.__name__ for module in modules] == [ + "InstalledPackageModule", + "PathLoadedModule", + ] + + +def test_list_modules_shows_module_sources(tmp_path, caplog): + module_path = _write_custom_module( + tmp_path / "custom.py", + "SourcedBackupModule", + (("ios", "check-backup"),), + ) + file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest() + custom_modules = module_loader.load_custom_modules([str(module_path)]) + + from mvt.ios.cmd_check_backup import CmdIOSCheckBackup + + cmd = CmdIOSCheckBackup(target_path=str(tmp_path), custom_modules=custom_modules) + cmd.list_modules() + + assert f" - Modules from 'mvt@{MVT_VERSION}':" in caplog.text + assert ( + f" - Modules from '{module_path}' (sha256: {file_sha256}): SourcedBackupModule" + in caplog.text + ) + + +def test_builtin_module_origin(): + from mvt.ios.modules.backup import BACKUP_MODULES + + origin = module_loader.get_module_origin(BACKUP_MODULES[0]) + + assert origin.kind == "builtin" + assert origin.name == "mvt" + assert origin.version == MVT_VERSION + + +def test_installed_module_origin(monkeypatch): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + + modules = module_loader.load_custom_modules() + + origin = module_loader.get_module_origin(modules[0]) + assert origin.kind == "package" + assert origin.name == "test-modules" + + +def test_distribution_commit_read_from_direct_url(): + class FakeDistribution: + def read_text(self, filename): + assert filename == "direct_url.json" + return json.dumps( + { + "url": "https://github.com/example/example-modules", + "vcs_info": {"commit_id": "abc1234", "vcs": "git"}, + } + ) + + assert module_loader._distribution_commit(FakeDistribution()) == "abc1234" + + +def test_command_log_records_loaded_modules(tmp_path): + (tmp_path / "Manifest.db").touch() + (tmp_path / "Info.plist").touch() + module_path = _write_custom_module( + tmp_path / "custom.py", + "AuditedRunModule", + (("ios", "check-backup"),), + slug="audited_run_module", + ) + file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest() + output_path = tmp_path / "out" + + result = CliRunner().invoke( + check_backup, + [ + "--module", + "AuditedRunModule", + "--load-module", + str(module_path), + "--output", + str(output_path), + str(tmp_path), + ], + ) + + assert result.exit_code == 0 + command_log = (output_path / "command.log").read_text(encoding="utf-8") + assert ( + f"Loaded 1 check-backup modules from '{module_path}' " + f"(sha256: {file_sha256}): AuditedRunModule" in command_log + ) + + class NestedBugreportModule(MVTModule): supported_commands = (("android", "check-bugreport"),)