mirror of
https://github.com/mvt-project/mvt.git
synced 2026-10-22 23:28:23 +02:00
Compare commits
29
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
348b47db35 | ||
|
|
b768d5b86c | ||
|
|
c4215ec163 | ||
|
|
36d1b27c4c | ||
|
|
6070fc06fb | ||
|
|
c95e6659c6 | ||
|
|
f8456059db | ||
|
|
efbab29f94 | ||
|
|
ecc22f54b7 | ||
|
|
90fc0ae351 | ||
|
|
dab82e38d0 | ||
|
|
4e031cd43f | ||
|
|
dac4acb180 | ||
|
|
dd8bd2cb01 | ||
|
|
30c11f68c7 | ||
|
|
a92ec7f963 | ||
|
|
10a5bccbe0 | ||
|
|
3b21c9347c | ||
|
|
0fdc2c34b0 | ||
|
|
0ee25edf0a | ||
|
|
fa24b5465b | ||
|
|
683b8ba133 | ||
|
|
7b64463727 | ||
|
|
47fd771b2a | ||
|
|
8123db26d9 | ||
|
|
d92a60c9be | ||
|
|
0b48d9fe1d | ||
|
|
067f053627 | ||
|
|
93b7fb5232 |
No files matched your search
@@ -11,7 +11,7 @@ jobs:
|
||||
name: Add issue to project
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/add-to-project@v0.5.0
|
||||
- uses: actions/add-to-project@v2
|
||||
with:
|
||||
# You can target a project in a different organization
|
||||
# to the issue
|
||||
|
||||
@@ -11,13 +11,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v10.0.0
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install Dependencies
|
||||
|
||||
@@ -36,10 +36,10 @@ jobs:
|
||||
tag-suffix: "-android"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
flavor: |
|
||||
@@ -59,21 +59,24 @@ jobs:
|
||||
type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }}
|
||||
type=sha,suffix=${{ matrix.platform.tag-suffix }}
|
||||
type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }}
|
||||
# Register emulators so Buildx can build the ARM64 images on the AMD64 runner.
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
# This step sets up some additional capabilities to generate the provenance and sbom attestations
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
|
||||
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
|
||||
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
|
||||
- name: Build and push Docker image
|
||||
id: push
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
file: ${{ matrix.platform.dockerfile }}
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
|
||||
@@ -12,13 +12,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@v7
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v10.0.0
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install Dependencies
|
||||
|
||||
@@ -15,13 +15,13 @@ jobs:
|
||||
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v10.0.0
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install Python dependencies
|
||||
|
||||
@@ -11,10 +11,12 @@ jobs:
|
||||
update-ios-version:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v7
|
||||
- name: Install script dependencies
|
||||
run: python -m pip install "packaging==26.3"
|
||||
- name: Run script to fetch latest iOS releases from Apple RSS feed.
|
||||
run: python3 .github/workflows/scripts/update-ios-releases.py
|
||||
run: python .github/workflows/scripts/update-ios-releases.py
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
|
||||
+13
-2
@@ -119,12 +119,23 @@ ARG DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y \
|
||||
adb \
|
||||
binutils \
|
||||
default-jre-headless \
|
||||
file \
|
||||
jq \
|
||||
less \
|
||||
libcurl4 \
|
||||
libimage-exiftool-perl \
|
||||
libssl3 \
|
||||
libusb-1.0-0 \
|
||||
moreutils \
|
||||
p7zip-full \
|
||||
python3 \
|
||||
sqlite3
|
||||
ripgrep \
|
||||
sqlite3 \
|
||||
tree \
|
||||
unzip \
|
||||
xxd
|
||||
COPY --from=build-libplist /build /
|
||||
COPY --from=build-libimobiledevice-glue /build /
|
||||
COPY --from=build-libtatsu /build /
|
||||
@@ -144,7 +155,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2
|
||||
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
|
||||
|
||||
# Generate adb key folder
|
||||
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc
|
||||
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc
|
||||
RUN mkdir /root/.android
|
||||
|
||||
# Setup investigations environment
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2
|
||||
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
|
||||
|
||||
# Generate adb key folder
|
||||
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc
|
||||
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc
|
||||
RUN mkdir /root/.android
|
||||
|
||||
ENTRYPOINT [ "/usr/local/bin/mvt-android" ]
|
||||
@@ -58,29 +58,31 @@ For alternative installation options and known issues, please refer to the [docu
|
||||
|
||||
## Usage
|
||||
|
||||
MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
|
||||
MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion`, `plugins` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
|
||||
|
||||
### Shell completion
|
||||
|
||||
MVT can generate shell completion scripts for Bash, Zsh, and Fish:
|
||||
MVT can generate a shell completion script for Bash, Zsh, and Fish which covers `mvt`, `mvt-ios` and `mvt-android`:
|
||||
|
||||
```bash
|
||||
mvt-ios completion
|
||||
mvt-android completion
|
||||
mvt completion
|
||||
```
|
||||
|
||||
The commands print setup instructions by default. To generate a completion script directly, pass the shell name:
|
||||
The command prints setup instructions by default. To generate the completion script directly, pass the shell name:
|
||||
|
||||
```bash
|
||||
mvt-ios completion bash
|
||||
mvt-android completion zsh
|
||||
mvt completion bash
|
||||
```
|
||||
|
||||
MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details.
|
||||
Module-running `check-*` commands can load custom Python modules with
|
||||
`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the
|
||||
|
||||
Plugin packages extend MVT with additional forensic modules, which run inside
|
||||
the `check-*` commands, and with top-level commands on `mvt`, `mvt-ios` and
|
||||
`mvt-android`. See the
|
||||
[development documentation](https://docs.mvt.re/en/latest/development/) for
|
||||
details.
|
||||
writing and installing them, and the
|
||||
[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/)
|
||||
for the entry points a package registers commands in.
|
||||
|
||||
|
||||
## License
|
||||
|
||||
+22
-22
@@ -6,61 +6,61 @@ Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fi
|
||||
|
||||
To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using.
|
||||
|
||||
The following describes how to generate the command completion scripts and add them to your shell configuration.
|
||||
`mvt completion` generates one script which covers `mvt`, `mvt-ios` and `mvt-android`. The following describes how to generate that script and add it to your shell configuration.
|
||||
|
||||
> **Note: You will need to start a new shell for the changes to take effect.**
|
||||
|
||||
### For Bash
|
||||
|
||||
```bash
|
||||
# Generate bash completion scripts
|
||||
mvt-ios completion bash > ~/.mvt-ios-complete.bash
|
||||
mvt-android completion bash > ~/.mvt-android-complete.bash
|
||||
# Generate the bash completion script
|
||||
mvt completion bash > ~/.mvt-complete.bash
|
||||
```
|
||||
|
||||
Add the following to `~/.bashrc`:
|
||||
```bash
|
||||
# source mvt completion scripts
|
||||
[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash
|
||||
[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash
|
||||
# source the mvt completion script
|
||||
[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash
|
||||
```
|
||||
|
||||
### For Zsh
|
||||
|
||||
```bash
|
||||
# Generate zsh completion scripts
|
||||
mvt-ios completion zsh > ~/.mvt-ios-complete.zsh
|
||||
mvt-android completion zsh > ~/.mvt-android-complete.zsh
|
||||
# Generate the zsh completion script
|
||||
mvt completion zsh > ~/.mvt-complete.zsh
|
||||
```
|
||||
|
||||
Add the following to `~/.zshrc`:
|
||||
```bash
|
||||
# source mvt completion scripts
|
||||
[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh
|
||||
[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh
|
||||
# source the mvt completion script
|
||||
[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh
|
||||
```
|
||||
|
||||
### For Fish
|
||||
|
||||
```bash
|
||||
# Generate fish completion scripts
|
||||
mkdir -p ~/.config/fish/completions
|
||||
mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish
|
||||
mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish
|
||||
# Generate the fish completion script
|
||||
mkdir -p ~/.config/fish/conf.d
|
||||
mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish
|
||||
```
|
||||
|
||||
Fish loads completion files from `~/.config/fish/completions` automatically.
|
||||
Fish loads the files in `~/.config/fish/conf.d` automatically.
|
||||
|
||||
### Automatic Installation
|
||||
|
||||
MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`:
|
||||
|
||||
```bash
|
||||
mvt-ios completion bash --install
|
||||
mvt-android completion bash --install
|
||||
mvt completion bash --install
|
||||
```
|
||||
|
||||
Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`.
|
||||
Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/conf.d` and changes no shell configuration.
|
||||
|
||||
!!! note
|
||||
|
||||
Earlier versions generated one script per command, with `mvt-ios completion`
|
||||
and `mvt-android completion`. Files written by them keep working. When you
|
||||
switch to the single script, remove the old files and the lines which load
|
||||
them from your shell configuration.
|
||||
|
||||
For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion).
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
# Development
|
||||
|
||||
The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software.
|
||||
|
||||
## Local environment
|
||||
|
||||
MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run:
|
||||
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can
|
||||
be run automatically with:
|
||||
|
||||
```bash
|
||||
make check
|
||||
```
|
||||
|
||||
Run these tests before making new commits or opening pull requests.
|
||||
|
||||
## Module dependencies
|
||||
|
||||
Modules can require other modules to run first by declaring their classes in
|
||||
`dependencies`. The command runner uses a stable topological ordering, so the
|
||||
existing module list order is preserved wherever dependency constraints allow.
|
||||
|
||||
```python
|
||||
class DependentModule(MVTModule):
|
||||
dependencies = (PrerequisiteModule,)
|
||||
|
||||
def run(self):
|
||||
prerequisite_results = self.get_dependency_results(PrerequisiteModule)
|
||||
```
|
||||
|
||||
Selecting a single module also runs its transitive dependencies. If a dependency
|
||||
is unavailable or the dependency graph contains a cycle, the command logs a
|
||||
warning and does not run any modules.
|
||||
|
||||
## Custom modules
|
||||
|
||||
Module-running `check-*` commands can load custom modules from Python files that
|
||||
are not installed as part of MVT. Load one file with:
|
||||
|
||||
```bash
|
||||
mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup
|
||||
```
|
||||
|
||||
You can also load a folder. MVT loads non-hidden top-level `*.py` files in
|
||||
sorted order and skips `__init__.py`:
|
||||
|
||||
```bash
|
||||
mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump
|
||||
```
|
||||
|
||||
Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This
|
||||
folder is loaded before any `--load-module` path:
|
||||
|
||||
```bash
|
||||
MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip
|
||||
```
|
||||
|
||||
Custom modules are normal `MVTModule` subclasses:
|
||||
|
||||
```python
|
||||
from mvt.common.module import MVTModule
|
||||
|
||||
|
||||
class ExampleCustomModule(MVTModule):
|
||||
supported_commands = (("ios", "check-backup"), ("ios", "check-fs"))
|
||||
slug = "example_custom_module"
|
||||
|
||||
def run(self):
|
||||
self.results = [{"message": "custom module ran"}]
|
||||
|
||||
def check_indicators(self):
|
||||
pass
|
||||
|
||||
def serialize(self, result):
|
||||
return None
|
||||
```
|
||||
|
||||
Use `supported_commands` to declare the platform/command pairs a module
|
||||
supports. Empty `supported_commands` means the module will not run and MVT logs
|
||||
a warning. This explicit declaration is required for every command. Supported
|
||||
pairs are:
|
||||
|
||||
```python
|
||||
("ios", "check-backup")
|
||||
("ios", "check-fs")
|
||||
("ios", "check-iocs")
|
||||
("ios", "check-sysdiagnose")
|
||||
("android", "check-backup")
|
||||
("android", "check-bugreport")
|
||||
("android", "check-androidqf")
|
||||
("android", "check-intrusion-logs")
|
||||
("android", "check-iocs")
|
||||
```
|
||||
|
||||
Custom modules can depend on existing MVT module classes. Dependencies are
|
||||
resolved with the same ordering logic as built-in modules, and custom modules
|
||||
are appended after built-ins before ordering:
|
||||
|
||||
```python
|
||||
from mvt.common.module import MVTModule
|
||||
from mvt.ios.modules.backup.manifest import Manifest
|
||||
|
||||
|
||||
class DependentCustomModule(MVTModule):
|
||||
supported_commands = (("ios", "check-backup"),)
|
||||
dependencies = (Manifest,)
|
||||
|
||||
def run(self):
|
||||
manifest_results = self.get_dependency_results(Manifest)
|
||||
self.results = [{"manifest_entries": len(manifest_results)}]
|
||||
```
|
||||
|
||||
## Profiling
|
||||
|
||||
Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be
|
||||
take to avoid inefficient code paths as we add new modules.
|
||||
|
||||
MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable.
|
||||
|
||||
```bash
|
||||
MVT_PROFILE=1 dev/mvt-ios check-backup test_backup
|
||||
```
|
||||
|
||||
Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT.
|
||||
@@ -0,0 +1,144 @@
|
||||
# Custom CLI Commands
|
||||
|
||||
MVT can load additional top-level commands into `mvt`, `mvt-ios` and
|
||||
`mvt-android`. A command package chooses which of the three each of its
|
||||
commands is added to.
|
||||
Custom commands are different from [custom forensic modules](index.md#custom-modules):
|
||||
commands add new CLI operations, while modules add analysis steps to existing
|
||||
`check-*` commands.
|
||||
|
||||
!!! warning
|
||||
|
||||
Custom commands run as trusted Python code inside the MVT process. Install
|
||||
or load commands only from sources you trust. MVT does not sandbox
|
||||
third-party commands, and the MVT maintainers do not maintain them.
|
||||
|
||||
## Install a Command Package
|
||||
|
||||
Python packages can register a Click command or group on one or more of the MVT CLIs.
|
||||
A minimal package can expose this command from `my_mvt_plugin.py`:
|
||||
|
||||
```python
|
||||
import click
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("path", type=click.Path(exists=True))
|
||||
def summarize(path):
|
||||
"""Summarize an acquisition."""
|
||||
click.echo(f"Summarizing {path}")
|
||||
```
|
||||
|
||||
Register the object in the package's `pyproject.toml`. The entry-point name is
|
||||
the command users invoke:
|
||||
|
||||
```toml
|
||||
[project.entry-points."mvt.ios.cli_plugins"]
|
||||
summarize = "my_mvt_plugin:summarize"
|
||||
|
||||
[project.entry-points."mvt.android.cli_plugins"]
|
||||
summarize = "my_mvt_plugin:summarize"
|
||||
```
|
||||
|
||||
Each entry-point group adds the command to one CLI: `mvt.ios.cli_plugins` to
|
||||
`mvt-ios`, `mvt.android.cli_plugins` to `mvt-android` and `mvt.cli_plugins` to
|
||||
`mvt`. Register the command in the group of every CLI which should offer it: a
|
||||
platform-specific command belongs in one platform group, and a command which
|
||||
handles acquisitions of both platforms, as above, in both. After installing the
|
||||
package in the same environment as MVT, the command appears directly in those
|
||||
CLIs:
|
||||
|
||||
```bash
|
||||
mvt-ios summarize ./ios-backup
|
||||
mvt-android summarize ./androidqf-output
|
||||
```
|
||||
|
||||
For a `pipx` installation of MVT, inject the plugin into MVT's environment:
|
||||
|
||||
```bash
|
||||
pipx inject mvt my-mvt-plugin
|
||||
```
|
||||
|
||||
When MVT is installed in an active virtual environment, install the plugin with
|
||||
`pip` in that environment. `mvt plugins list` shows the installed packages and
|
||||
the commands they add, see [Managing Plugins](plugins.md).
|
||||
|
||||
Command packages that need their own settings, such as an API key, should store
|
||||
them in a namespaced [plugin configuration file](plugin_configuration.md)
|
||||
rather than in MVT's own `config.yaml`.
|
||||
|
||||
### Commands on `mvt`
|
||||
|
||||
The `mvt` command hosts what belongs to neither platform: `version`,
|
||||
`completion`, `plugins` and `download-iocs`. A plugin command which is not
|
||||
about the acquisition of one platform, such as one which configures the plugin
|
||||
or synchronizes the indicators it uses, belongs there too, in the
|
||||
`mvt.cli_plugins` group:
|
||||
|
||||
```toml
|
||||
[project.entry-points."mvt.cli_plugins"]
|
||||
my-plugin = "my_mvt_plugin:my_plugin"
|
||||
```
|
||||
|
||||
Commands in this group are added to `mvt` only, so this one is invoked as
|
||||
`mvt my-plugin`. A command on `mvt` has nothing but its name to say which
|
||||
plugin it belongs to, so name it after the plugin, and make it a Click group
|
||||
when the plugin has several operations to offer, such as
|
||||
`mvt my-plugin configure`.
|
||||
|
||||
## Developing a Command Locally
|
||||
|
||||
A package is how a command is distributed. While a command is being written,
|
||||
MVT can load it straight from its file instead, so the package need not be
|
||||
reinstalled after every change; an editable install of the package does the
|
||||
same through its entry points. Create a Python file that exports one Click
|
||||
command or group named `cli`:
|
||||
|
||||
```python
|
||||
import click
|
||||
|
||||
|
||||
@click.command("case-summary")
|
||||
@click.argument("path", type=click.Path(exists=True))
|
||||
def cli(path):
|
||||
"""Summarize a case directory."""
|
||||
click.echo(f"Summarizing {path}")
|
||||
```
|
||||
|
||||
Pass the file before the custom command name:
|
||||
|
||||
```bash
|
||||
mvt-ios --load-command ./case_summary.py case-summary ./ios-backup
|
||||
```
|
||||
|
||||
`--load-command` can be repeated and also accepts a folder. MVT loads
|
||||
non-hidden top-level `*.py` files in sorted order and skips `__init__.py`.
|
||||
Every loaded file must export one `cli` object.
|
||||
|
||||
To load a file or folder on every invocation, set the environment variable of
|
||||
the CLI the commands belong on. Like the entry-point groups, each variable adds
|
||||
its commands to one CLI only:
|
||||
|
||||
```bash
|
||||
export MVT_CUSTOM_COMMANDS=./commands
|
||||
export MVT_IOS_CUSTOM_COMMANDS=./ios_commands
|
||||
export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands
|
||||
```
|
||||
|
||||
## Naming and Errors
|
||||
|
||||
Built-in MVT commands cannot be replaced. External command names must also be
|
||||
unique on each CLI; when installed packages or environment paths collide, MVT
|
||||
keeps the first command and logs a warning. The environment path of a CLI is
|
||||
registered before its installed packages, so a command loaded from there wins a
|
||||
collision with a package. A collision from an explicit `--load-command` is a
|
||||
usage error.
|
||||
|
||||
A package entry point or environment command that cannot be imported appears
|
||||
as a marked broken command without preventing other MVT commands from working.
|
||||
Invoke that command to see its package or file source and the underlying error.
|
||||
An invalid command supplied explicitly with `--load-command` fails immediately
|
||||
with a usage error.
|
||||
|
||||
Installed command packages use the entry-point name as the CLI command name.
|
||||
The entry point must resolve to a `click.Command` or `click.Group`.
|
||||
@@ -0,0 +1,232 @@
|
||||
# Development
|
||||
|
||||
The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software.
|
||||
|
||||
## Local environment
|
||||
|
||||
MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run:
|
||||
|
||||
```bash
|
||||
make install
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can
|
||||
be run automatically with:
|
||||
|
||||
```bash
|
||||
make check
|
||||
```
|
||||
|
||||
Run these tests before making new commits or opening pull requests.
|
||||
|
||||
## Module dependencies
|
||||
|
||||
Modules can require other modules to run first by declaring their classes in
|
||||
`dependencies`. The command runner uses a stable topological ordering, so the
|
||||
existing module list order is preserved wherever dependency constraints allow.
|
||||
|
||||
```python
|
||||
class DependentModule(MVTModule):
|
||||
dependencies = (PrerequisiteModule,)
|
||||
|
||||
def run(self):
|
||||
prerequisite_results = self.get_dependency_results(PrerequisiteModule)
|
||||
```
|
||||
|
||||
Selecting a single module also runs its transitive dependencies.
|
||||
|
||||
A module can only depend on modules the command it runs in also has. When a
|
||||
declared dependency is not among them, the command logs a warning naming the
|
||||
module and the missing dependency, skips that module and everything depending
|
||||
on it, and runs the rest of the analysis. Selecting such a module with
|
||||
`--module` therefore leaves nothing to run, which the warning explains.
|
||||
|
||||
A cycle in the dependency graph is a programming error rather than a
|
||||
configuration problem: the command logs a warning and runs no modules at all.
|
||||
|
||||
## Custom modules
|
||||
|
||||
Module-running `check-*` commands can load custom modules from Python files that
|
||||
are not installed as part of MVT. Load one file with:
|
||||
|
||||
```bash
|
||||
mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup
|
||||
```
|
||||
|
||||
You can also load a folder. MVT loads non-hidden top-level `*.py` files in
|
||||
sorted order and skips `__init__.py`:
|
||||
|
||||
```bash
|
||||
mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump
|
||||
```
|
||||
|
||||
Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This
|
||||
folder is loaded before any `--load-module` path:
|
||||
|
||||
```bash
|
||||
MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip
|
||||
```
|
||||
|
||||
Custom modules are normal `MVTModule` subclasses:
|
||||
|
||||
```python
|
||||
from mvt.common.module import MVTModule
|
||||
|
||||
|
||||
class ExampleCustomModule(MVTModule):
|
||||
supported_commands = (("ios", "check-backup"), ("ios", "check-fs"))
|
||||
slug = "example_custom_module"
|
||||
|
||||
def run(self):
|
||||
self.results = [{"message": "custom module ran"}]
|
||||
|
||||
def check_indicators(self):
|
||||
pass
|
||||
|
||||
def serialize(self, result):
|
||||
return None
|
||||
```
|
||||
|
||||
Use `supported_commands` to declare the platform/command pairs a module
|
||||
supports. Empty `supported_commands` means the module will not run and MVT logs
|
||||
a warning. This explicit declaration is required for every command. Supported
|
||||
pairs are:
|
||||
|
||||
```python
|
||||
("ios", "check-backup")
|
||||
("ios", "check-fs")
|
||||
("ios", "check-iocs")
|
||||
("ios", "check-sysdiagnose")
|
||||
("android", "check-backup")
|
||||
("android", "check-bugreport")
|
||||
("android", "check-androidqf")
|
||||
("android", "check-intrusion-logs")
|
||||
("android", "check-iocs")
|
||||
```
|
||||
|
||||
Custom modules can depend on existing MVT module classes. Dependencies are
|
||||
resolved with the same ordering logic as built-in modules, and custom modules
|
||||
are appended after built-ins before ordering:
|
||||
|
||||
```python
|
||||
from mvt.common.module import MVTModule
|
||||
from mvt.ios.modules.backup.manifest import Manifest
|
||||
|
||||
|
||||
class DependentCustomModule(MVTModule):
|
||||
supported_commands = (("ios", "check-backup"),)
|
||||
dependencies = (Manifest,)
|
||||
|
||||
def run(self):
|
||||
manifest_results = self.get_dependency_results(Manifest)
|
||||
self.results = [{"manifest_entries": len(manifest_results)}]
|
||||
```
|
||||
|
||||
## Installed module packages
|
||||
|
||||
Python packages can register modules so they load automatically in every
|
||||
module-running `check-*` command, without `--load-module` or
|
||||
`MVT_CUSTOM_MODULES`. Register an entry point in the `mvt.modules` group in
|
||||
the package's `pyproject.toml`:
|
||||
|
||||
```toml
|
||||
[project.entry-points."mvt.modules"]
|
||||
mvt-plugin-amnesty-custom = "mvt_plugin_amnesty_custom:get_modules"
|
||||
```
|
||||
|
||||
The entry point must resolve to an iterable of `MVTModule` subclasses, or to
|
||||
a callable returning one:
|
||||
|
||||
```python
|
||||
from mvt.common.module import MVTModule
|
||||
|
||||
|
||||
class PackagedModule(MVTModule):
|
||||
supported_commands = (("ios", "check-backup"),)
|
||||
|
||||
def run(self):
|
||||
self.results = [{"message": "packaged module ran"}]
|
||||
|
||||
|
||||
def get_modules() -> list[type[MVTModule]]:
|
||||
return [PackagedModule]
|
||||
```
|
||||
|
||||
Installed modules follow the same rules as other custom modules: each module
|
||||
must declare `supported_commands`, and dependencies are resolved with the
|
||||
standard ordering logic. A broken entry point is skipped with a warning and
|
||||
does not prevent MVT from running. As with custom commands, installed module
|
||||
packages run as trusted code inside the MVT process, so install only packages
|
||||
from sources you trust.
|
||||
|
||||
For a `pipx` installation of MVT, inject the package into MVT's environment:
|
||||
|
||||
```bash
|
||||
pipx inject mvt mvt-plugin-amnesty-custom
|
||||
```
|
||||
|
||||
Module packages that need their own settings, such as an API key, should store
|
||||
them in a namespaced [plugin configuration file](plugin_configuration.md)
|
||||
rather than in MVT's own `config.yaml`.
|
||||
|
||||
### Naming module packages
|
||||
|
||||
Name module packages `mvt-plugin-<name>` (import package `mvt_plugin_<name>`),
|
||||
and include the name of the publishing organization or author so packages from
|
||||
different groups do not collide: for example, Amnesty International's custom
|
||||
modules would be distributed as `mvt-plugin-amnesty-custom` with the import
|
||||
package `mvt_plugin_amnesty_custom`.
|
||||
|
||||
The prefix makes module packages easy to find on PyPI and keeps their import
|
||||
names from clashing with unrelated Python packages. It is a convention, not a
|
||||
technical requirement: modules load through the `mvt.modules` entry point
|
||||
regardless of what the package is called, and existing packages with other
|
||||
names keep working. Note that the prefix is also not a mark of authenticity —
|
||||
anyone can publish a package with any available name, so vet a module package
|
||||
and its publisher before installing it, whatever it is called.
|
||||
|
||||
### Module logging
|
||||
|
||||
Modules log through `self.log`, and MVT names the logger for where the module
|
||||
came from. MVT's own modules log under their dotted path (for example
|
||||
`mvt.ios.modules.mixed.whatsapp`). Everything external is namespaced under
|
||||
`mvt.ext` to keep it visually distinct from built-in modules and isolated from
|
||||
MVT's internal logger tree:
|
||||
|
||||
- Installed packages log under `mvt.ext.<package>`, with the `mvt_plugin_`
|
||||
prefix stripped: modules in `mvt_plugin_amnesty_custom` log as
|
||||
`mvt.ext.amnesty_custom.*`.
|
||||
- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as
|
||||
`mvt.ext.<file name>`.
|
||||
|
||||
## Auditing loaded modules
|
||||
|
||||
Because installed module packages load automatically, MVT records where every
|
||||
module came from:
|
||||
|
||||
- `--list-modules` groups the available modules by source: MVT itself
|
||||
(with its version), each installed package (with its version and, when
|
||||
installed directly from a repository, the commit), and each file loaded
|
||||
with `--load-module` or `MVT_CUSTOM_MODULES` (with the SHA-256 hash of the
|
||||
file).
|
||||
- When a command runs with an `--output` folder, the `command.log` file
|
||||
records one line per module source with the source's version or hash and
|
||||
the list of modules loaded from it.
|
||||
- `mvt plugins list` lists the installed packages, where each of them
|
||||
was installed from and how many modules it contributes, see
|
||||
[Managing Plugins](plugins.md).
|
||||
|
||||
## Profiling
|
||||
|
||||
Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be
|
||||
take to avoid inefficient code paths as we add new modules.
|
||||
|
||||
MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable.
|
||||
|
||||
```bash
|
||||
MVT_PROFILE=1 dev/mvt-ios check-backup test_backup
|
||||
```
|
||||
|
||||
Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT.
|
||||
@@ -0,0 +1,149 @@
|
||||
# Plugin Configuration
|
||||
|
||||
Plugin packages that add [custom CLI commands](custom_commands.md) or
|
||||
[modules](index.md#installed-module-packages) often need to store their
|
||||
own settings, such as an API key, a server URL or the timestamp of the last
|
||||
synchronization. MVT provides a namespaced settings base class so each plugin
|
||||
keeps its configuration in its own file, and a data folder for anything else a
|
||||
plugin needs to keep on disk.
|
||||
|
||||
!!! warning
|
||||
|
||||
Do not write plugin settings to MVT's own `config.yaml`. MVT rewrites that
|
||||
file with the settings it knows about every time it starts, so any other
|
||||
section is deleted.
|
||||
|
||||
## Where Settings Are Stored
|
||||
|
||||
Each plugin gets one YAML file in a `plugins` folder next to MVT's own
|
||||
configuration:
|
||||
|
||||
```
|
||||
~/.config/mvt/plugins/<plugin name>.yaml
|
||||
```
|
||||
|
||||
The exact parent folder follows the platform convention used for MVT's
|
||||
`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use
|
||||
`mvt.common.plugin_config.plugin_config_path()` instead of building the path by
|
||||
hand.
|
||||
|
||||
Plugin names must be lowercase and may only contain letters, digits and dashes,
|
||||
matching the `mvt-plugin-<name>` package naming convention. MVT creates the
|
||||
`plugins` folder with `0700` permissions and writes the settings files with
|
||||
`0600` permissions, because they commonly hold credentials. Files are written
|
||||
through a temporary file and moved into place, so an interrupted save never
|
||||
leaves a partially written settings file behind.
|
||||
|
||||
## Plugin Data Folder
|
||||
|
||||
Everything else a plugin keeps on disk, such as a cache, a downloaded artifact
|
||||
or synchronization state, belongs in the folder returned by the `data_folder()`
|
||||
class method of the plugin's settings class, or by
|
||||
`mvt.common.plugin_config.plugin_data_folder()` called with the plugin name if
|
||||
the plugin has no settings class:
|
||||
|
||||
```
|
||||
~/.local/share/mvt/plugin-data/<plugin name>/ # Linux
|
||||
~/Library/Application Support/mvt/plugin-data/<plugin name>/ # macOS
|
||||
```
|
||||
|
||||
The folder sits beside MVT's own data, such as the downloaded indicators. It is
|
||||
created if it is missing, with `0700` permissions. Asking for it again returns
|
||||
the same path and leaves the contents alone, so a plugin can ask for it every
|
||||
time it needs the folder. `ExamplePluginSettings` below is the settings class
|
||||
defined in the next section:
|
||||
|
||||
```python
|
||||
import os
|
||||
|
||||
|
||||
def cache_path() -> str:
|
||||
folder = ExamplePluginSettings.data_folder()
|
||||
return os.path.join(folder, "virustotal_lookups_cache.json")
|
||||
```
|
||||
|
||||
A plugin which has no settings class calls
|
||||
`plugin_data_folder("example-plugin")` instead.
|
||||
|
||||
Do not fall back on a path of your own such as `~/.cache/example-plugin`: it
|
||||
is a Linux-only convention, and MVT will not create it for you.
|
||||
|
||||
## Defining Plugin Settings
|
||||
|
||||
Subclass `MVTPluginSettings`, set `plugin_name` and declare typed fields with
|
||||
defaults:
|
||||
|
||||
```python
|
||||
from typing import Optional
|
||||
|
||||
from mvt.common.plugin_config import MVTPluginSettings
|
||||
|
||||
|
||||
class ExamplePluginSettings(MVTPluginSettings):
|
||||
plugin_name = "example-plugin"
|
||||
|
||||
API_KEY: Optional[str] = None
|
||||
MAX_RESULTS: int = 25
|
||||
LAST_SYNC: Optional[str] = None
|
||||
```
|
||||
|
||||
`load()` returns the current settings and `save()` writes them back:
|
||||
|
||||
```python
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import click
|
||||
|
||||
|
||||
def sync():
|
||||
settings = ExamplePluginSettings.load()
|
||||
if not settings.API_KEY:
|
||||
raise click.ClickException(
|
||||
"No API key configured. Set MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY or "
|
||||
"run 'example-plugin configure'."
|
||||
)
|
||||
|
||||
settings.LAST_SYNC = datetime.now(timezone.utc).isoformat()
|
||||
settings.save()
|
||||
```
|
||||
|
||||
A missing settings file is not an error: the plugin then runs on the field
|
||||
defaults and on whatever the environment provides. `save()` only persists the
|
||||
values that differ from the defaults, and it never touches MVT's `config.yaml`.
|
||||
A settings file that cannot be parsed, or that does not hold a mapping of
|
||||
setting names to values, raises a `PluginConfigLoadError` naming the file.
|
||||
|
||||
Every subclass that sets its own `plugin_name` gets its own file and its own
|
||||
environment namespace. A subclass that does not redefine `plugin_name` inherits
|
||||
it, and therefore shares the file and the environment variables of its parent
|
||||
class.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Every field can also be set with an environment variable. The prefix is
|
||||
`MVT_PLUGIN_`, followed by the plugin name upper-cased with dashes replaced by
|
||||
underscores, followed by the field name. For the example above:
|
||||
|
||||
```bash
|
||||
export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=...
|
||||
export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50
|
||||
```
|
||||
|
||||
Settings resolve in this order, from highest to lowest priority:
|
||||
|
||||
1. Arguments passed to the settings class directly, such as
|
||||
`ExamplePluginSettings(API_KEY="...")`
|
||||
2. Environment variables
|
||||
3. The plugin's YAML file
|
||||
4. The field defaults declared on the settings class
|
||||
|
||||
!!! tip
|
||||
|
||||
On shared or multi-user machines, prefer passing API keys through
|
||||
environment variables rather than saving them to the plugin file. `save()`
|
||||
skips every value that the environment currently supplies, so a credential
|
||||
provided that way is not copied into the settings file when a plugin saves
|
||||
an unrelated setting.
|
||||
|
||||
Unknown keys in a plugin's YAML file are ignored, so a settings file written by
|
||||
a newer version of a plugin does not break an older one.
|
||||
@@ -0,0 +1,116 @@
|
||||
# Managing Plugins
|
||||
|
||||
Plugin packages extend MVT with additional
|
||||
[forensic modules](index.md#custom-modules) and
|
||||
[CLI commands](custom_commands.md). Because installed packages load
|
||||
automatically, `mvt plugins` audits what is installed and checks whether
|
||||
updates are available. The command lives on `mvt` only, although the packages
|
||||
it lists extend `mvt-ios` and `mvt-android` too.
|
||||
|
||||
## List Installed Plugins
|
||||
|
||||
```bash
|
||||
mvt plugins list
|
||||
```
|
||||
|
||||
```
|
||||
Installed MVT plugins
|
||||
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓
|
||||
┃ Name ┃ Version ┃ Origin ┃ Modules ┃ Commands ┃
|
||||
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩
|
||||
│ mvt-plugin-example │ 1.2.0 │ pypi │ 4 │ summarize │
|
||||
│ mvt-plugin-research │ 0.1.0 │ git+3f9a1c7d │ 2 │ - │
|
||||
│ mvt-plugin-local │ 0.0.1 │ local │ 1 │ triage │
|
||||
└─────────────────────┴─────────┴──────────────┴─────────┴───────────┘
|
||||
```
|
||||
|
||||
The origin records where each package was installed from: `pypi` for a package
|
||||
installed from a package index, `git+<commit>` for a package installed directly
|
||||
from a repository, and `local` for a package installed from a local folder or
|
||||
archive rather than from an index, including an editable development install.
|
||||
The last two columns show how many forensic modules the package contributes and
|
||||
which CLI commands it adds.
|
||||
|
||||
A plugin whose modules cannot be imported is listed with `error` in the
|
||||
`Modules` column rather than breaking the listing.
|
||||
|
||||
## Check for Updates
|
||||
|
||||
```bash
|
||||
mvt plugins check-updates
|
||||
```
|
||||
|
||||
```
|
||||
Plugin updates available:
|
||||
mvt-plugin-example 1.2.0 → 1.3.0
|
||||
Upgrade with: pip install -U mvt-plugin-example
|
||||
|
||||
MVT does not install plugin updates. Run the command above when you decide to
|
||||
upgrade.
|
||||
```
|
||||
|
||||
Packages installed from a package index are compared against the latest release
|
||||
published for them. A package which was never published, for example a plugin
|
||||
distributed only within an organization, is skipped silently.
|
||||
|
||||
!!! note
|
||||
|
||||
Packages shown with the `pypi` origin are compared against
|
||||
[PyPI](https://pypi.org), whichever index they were installed from. A
|
||||
plugin installed from a private index under a name which also exists on
|
||||
PyPI is therefore compared against the unrelated public package of that
|
||||
name. Give plugins published to a private index a name which is not taken
|
||||
on PyPI, and treat an unexpected update suggestion as a reason to check
|
||||
where the package would come from.
|
||||
|
||||
!!! warning
|
||||
|
||||
MVT never installs or upgrades a plugin itself, it only prints the command
|
||||
which does. Upgrading a plugin in the middle of an investigation changes
|
||||
the modules producing the results, and a plugin runs as trusted code inside
|
||||
the MVT process, so pulling in a new version is a decision for the analyst
|
||||
to make deliberately and not a side effect of running a check.
|
||||
|
||||
## Automatic Update Checks
|
||||
|
||||
MVT also reports available plugin updates in the banner printed when a command
|
||||
starts:
|
||||
|
||||
```
|
||||
MVT - Mobile Verification Toolkit
|
||||
|
||||
https://mvt.re
|
||||
Version: 2026.7.29
|
||||
|
||||
Plugin updates available:
|
||||
mvt-plugin-example 1.2.0 → 1.3.0 (pip install -U mvt-plugin-example)
|
||||
```
|
||||
|
||||
This check runs at most once every 12 hours. In between checks MVT prints the
|
||||
findings of the latest check without contacting anything, so a plugin update
|
||||
stays visible without a lookup on every command. The
|
||||
`mvt plugins check-updates` command checks immediately, regardless of when the
|
||||
last check happened.
|
||||
|
||||
The automatic check is skipped when the `--disable-update-check` option is
|
||||
used, when `NETWORK_ACCESS_ALLOWED` is disabled in the MVT configuration, and
|
||||
when no plugins are installed.
|
||||
|
||||
## Plugins Installed From a Repository
|
||||
|
||||
A plugin installed with `pip install "mvt-plugin-example @ git+<url>"` is
|
||||
checked by asking the remote repository which commit the installed revision
|
||||
points at now. MVT runs git and ssh in batch mode, so a repository which needs
|
||||
credentials MVT does not already have fails the check instead of prompting for
|
||||
them. The check is skipped silently when git is not available, when the
|
||||
repository cannot be reached, and when access to it is denied.
|
||||
|
||||
How the plugin was installed decides what an update means:
|
||||
|
||||
- A plugin installed from a branch is reported as outdated when the branch has
|
||||
moved past the installed commit.
|
||||
- A plugin installed from a specific commit or a tag is pinned. It is never
|
||||
reported as outdated, however far the branch it came from moves on.
|
||||
|
||||
Pinning a plugin to a commit or a tag is therefore the way to keep the modules
|
||||
used across an investigation stable.
|
||||
+15
-1
@@ -31,4 +31,18 @@ Test if the image was created successfully:
|
||||
docker run -it mvt
|
||||
```
|
||||
|
||||
If a prompt is spawned successfully, you can close it with `exit`.
|
||||
If a prompt is spawned successfully, you can close it with `exit`.
|
||||
|
||||
## Access an iOS device from Docker
|
||||
|
||||
On the Linux host, install and start [usbmuxd](https://github.com/libimobiledevice/usbmuxd), then connect and unlock the iOS device. The daemon exposes the device through the `/var/run/usbmuxd` socket.
|
||||
|
||||
Bind that socket into the container to let MVT communicate with the device:
|
||||
|
||||
```bash
|
||||
docker run -it \
|
||||
--mount type=bind,source=/var/run/usbmuxd,target=/var/run/usbmuxd \
|
||||
ghcr.io/mvt-project/mvt
|
||||
```
|
||||
|
||||
If you built the image from source, replace `ghcr.io/mvt-project/mvt` with `mvt`.
|
||||
+1
-1
@@ -9,7 +9,7 @@ Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forens
|
||||
It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors.
|
||||
|
||||
|
||||
In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results.
|
||||
In this documentation you will find instructions on how to install and run the `mvt-ios`, `mvt-android` and `mvt` commands, and guidance on how to interpret the extracted results.
|
||||
|
||||
## Resources
|
||||
|
||||
|
||||
+3
-3
@@ -64,7 +64,7 @@ It is recommended to try installing and running MVT from [Windows Subsystem Linu
|
||||
pipx install mvt
|
||||
```
|
||||
|
||||
You now should have the `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window.
|
||||
You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window.
|
||||
|
||||
### Installing from PyPI directly into a virtual environment
|
||||
You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool:
|
||||
@@ -84,7 +84,7 @@ source env/bin/activate
|
||||
pip install mvt
|
||||
```
|
||||
|
||||
The `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active.
|
||||
The `mvt`, `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active.
|
||||
|
||||
### Installing from git source with pipx
|
||||
If you want to have the latest features in development, you can install MVT directly from the source code in git.
|
||||
@@ -93,7 +93,7 @@ If you want to have the latest features in development, you can install MVT dire
|
||||
pipx install --force git+https://github.com/mvt-project/mvt.git
|
||||
```
|
||||
|
||||
You now should have the `mvt-ios` and `mvt-android` utilities installed.
|
||||
You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed.
|
||||
|
||||
**Notes:**
|
||||
1. The `--force` flag is necessary to force the reinstallation of the package.
|
||||
|
||||
+1
-1
@@ -71,7 +71,7 @@ So far MVT implements only a subset of [STIX2 specifications](https://docs.oasis
|
||||
- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2).
|
||||
- We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns.
|
||||
|
||||
You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT.
|
||||
You can automatically download the latest public indicator files with the command `mvt download-iocs`. The per-platform forms `mvt-ios download-iocs` and `mvt-android download-iocs` do the same thing. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT.
|
||||
|
||||
Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.
|
||||
|
||||
+15
-1
@@ -417,7 +417,21 @@ If indicators are provided through the command-line, they are checked against th
|
||||
Backup: :material-check:
|
||||
Full filesystem dump: :material-check:
|
||||
|
||||
This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*.
|
||||
This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*, along with one record per chat session (marked with `"record_type": "chat_session"`) containing the first and last interaction dates of each conversation. Chat sessions produce `chat_first_message` and `chat_last_message` timeline events, and group chats additionally produce a `group_created` event. A chat session's last-message date can postdate its newest stored message when the most recent messages in the chat were deleted.
|
||||
|
||||
Recent WhatsApp versions key 1:1 chat sessions by an opaque LID identifier rather than the contact's phone number. The module resolves these using the `ZWAPHONENUMBERLIDPAIR` table from the *LID.sqlite* database in the same app group (or from *ChatStorage.sqlite* itself in versions that store it there), populating `partner_resolved_phone_number` on chat session records and using the phone number in timeline events. Each LID-phone number pair is also extracted as a record (`"record_type": "lid_phone_number_pair"`) and produces a `lid_pair_recorded` timeline event marking when WhatsApp learned the association.
|
||||
|
||||
If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*.
|
||||
|
||||
---
|
||||
|
||||
### `whatsapp_contacts.json`
|
||||
|
||||
!!! info "Availability"
|
||||
Backup: :material-check:
|
||||
Full filesystem dump: :material-check:
|
||||
|
||||
This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extracts WhatsApp contact records from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ContactsV2.sqlite*, including each contact's phone number, WhatsApp and LID identifiers, and the per-contact disappearing messages timer, which is not recorded in *ChatStorage.sqlite*. Each timestamp stored on a contact record produces a timeline event: `disappearing_mode_set` (when the disappearing messages timer was last changed), `about_changed` (when the contact last changed their "about" text), `about_expiration` (when a timed "about" is scheduled to expire) and `contact_last_updated` (when the contact record was last updated).
|
||||
|
||||
This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup.
|
||||
|
||||
+5
-1
@@ -49,5 +49,9 @@ nav:
|
||||
- Check an Android Backup (SMS messages): "android/backup.md"
|
||||
- Check Android Intrusion Logs: "android/intrusion_logs.md"
|
||||
- Indicators of Compromise: "iocs.md"
|
||||
- Development: "development.md"
|
||||
- Development:
|
||||
- Development Instructions: "development/index.md"
|
||||
- Custom CLI Commands: "development/custom_commands.md"
|
||||
- Plugin Configuration: "development/plugin_configuration.md"
|
||||
- Managing Plugins: "development/plugins.md"
|
||||
- License: "license.md"
|
||||
+16
-15
@@ -17,25 +17,25 @@ classifiers = [
|
||||
"Programming Language :: Python",
|
||||
]
|
||||
dependencies = [
|
||||
"click==8.3.3",
|
||||
"rich==14.3.3",
|
||||
"click==8.4.2",
|
||||
"rich==15.0.0",
|
||||
"tld==0.13.2",
|
||||
"requests==2.33.1",
|
||||
"simplejson==3.20.2",
|
||||
"packaging==26.2",
|
||||
"requests==2.34.2",
|
||||
"simplejson==4.1.1",
|
||||
"packaging==26.3",
|
||||
"appdirs==1.4.4",
|
||||
"iOSbackup==0.9.925",
|
||||
"adb-shell[usb]==0.4.4",
|
||||
"libusb1==3.3.1",
|
||||
"cryptography==47.0.0",
|
||||
"libusb1==3.4.0",
|
||||
"cryptography==50.0.0",
|
||||
"PyYAML>=6.0.2",
|
||||
"pyahocorasick==2.3.1",
|
||||
"betterproto2==0.9.1",
|
||||
"pydantic==2.13.3",
|
||||
"pydantic-settings==2.14.0",
|
||||
"betterproto2==0.10.0",
|
||||
"pydantic==2.13.4",
|
||||
"pydantic-settings==2.15.0",
|
||||
"NSKeyedUnArchiver==1.5.2",
|
||||
"python-dateutil==2.9.0.post0",
|
||||
"tzdata==2026.2",
|
||||
"tzdata==2026.3",
|
||||
]
|
||||
requires-python = ">= 3.10"
|
||||
|
||||
@@ -44,8 +44,9 @@ homepage = "https://docs.mvt.re/en/latest/"
|
||||
repository = "https://github.com/mvt-project/mvt"
|
||||
|
||||
[project.scripts]
|
||||
mvt-ios = "mvt.ios:cli"
|
||||
mvt-android = "mvt.android:cli"
|
||||
mvt = "mvt.cli:main"
|
||||
mvt-ios = "mvt.ios:main"
|
||||
mvt-android = "mvt.android:main"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
@@ -65,9 +66,9 @@ dev = [
|
||||
docs = [
|
||||
"mkdocs==1.6.1",
|
||||
"mkdocs-autorefs==1.4.4",
|
||||
"mkdocs-material==9.7.6",
|
||||
"mkdocs-material==9.7.7",
|
||||
"mkdocs-material-extensions==1.3.1",
|
||||
"mkdocstrings==1.0.4",
|
||||
"mkdocstrings==1.0.6",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
||||
@@ -3,4 +3,4 @@
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
from .cli import cli
|
||||
from .cli import cli, main
|
||||
@@ -4,7 +4,7 @@
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from mvt.android.utils import ROOT_PACKAGES
|
||||
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
||||
@@ -71,6 +71,8 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
||||
"timestamp": "",
|
||||
"first_install_time": "",
|
||||
"last_update_time": "",
|
||||
"installer": "",
|
||||
"system": False,
|
||||
"permissions": list(),
|
||||
"requested_permissions": list(),
|
||||
}
|
||||
@@ -78,7 +80,14 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
||||
in_runtime_permissions = False
|
||||
in_declared_permissions = False
|
||||
in_requested_permissions = True
|
||||
current_user: Optional[int] = None
|
||||
first_install_times: Dict[Optional[int], str] = {}
|
||||
runtime_permissions: Dict[Optional[int], List[Dict[str, Any]]] = {}
|
||||
for line in output.splitlines():
|
||||
user_match = re.match(r"User (\d+):", line.strip())
|
||||
if user_match:
|
||||
current_user = int(user_match.group(1))
|
||||
|
||||
if in_install_permissions:
|
||||
if line.startswith(" " * 4) and not line.startswith(" " * 6):
|
||||
in_install_permissions = False
|
||||
@@ -102,7 +111,7 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
||||
if "granted=" in lineinfo[1]:
|
||||
granted = "granted=true" in lineinfo[1]
|
||||
|
||||
details["permissions"].append(
|
||||
runtime_permissions.setdefault(current_user, []).append(
|
||||
{"name": permission, "granted": granted, "type": "runtime"}
|
||||
)
|
||||
if in_declared_permissions:
|
||||
@@ -126,8 +135,12 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
||||
details["version_code"] = line.split("=", 1)[1].strip()
|
||||
elif line.strip().startswith("timeStamp="):
|
||||
details["timestamp"] = line.split("=")[1].strip()
|
||||
elif line.strip().startswith("installerPackageName="):
|
||||
details["installer"] = line.split("=", 1)[1].strip()
|
||||
elif line.strip().startswith("pkgFlags="):
|
||||
details["system"] = "SYSTEM" in line.split("=", 1)[1].split()
|
||||
elif line.strip().startswith("firstInstallTime="):
|
||||
details["first_install_time"] = line.split("=")[1].strip()
|
||||
first_install_times[current_user] = line.split("=", 1)[1].strip()
|
||||
elif line.strip().startswith("lastUpdateTime="):
|
||||
details["last_update_time"] = line.split("=")[1].strip()
|
||||
elif line.strip() == "install permissions:":
|
||||
@@ -139,6 +152,19 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
||||
elif line.strip() == "requested permissions:":
|
||||
in_requested_permissions = True
|
||||
|
||||
if 0 in first_install_times:
|
||||
details["first_install_time"] = first_install_times[0]
|
||||
elif None in first_install_times:
|
||||
details["first_install_time"] = first_install_times[None]
|
||||
elif first_install_times:
|
||||
details["first_install_time"] = next(iter(first_install_times.values()))
|
||||
|
||||
if 0 in runtime_permissions:
|
||||
details["permissions"].extend(runtime_permissions[0])
|
||||
elif None in runtime_permissions:
|
||||
details["permissions"].extend(runtime_permissions[None])
|
||||
elif runtime_permissions:
|
||||
details["permissions"].extend(next(iter(runtime_permissions.values())))
|
||||
return details
|
||||
|
||||
def parse_dumpsys_packages(self, output: str) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -34,6 +34,7 @@ TOMBSTONE_TEXT_KEY_MAPPINGS = {
|
||||
"signal": "signal_info",
|
||||
"code": "code",
|
||||
"Cause": "cause",
|
||||
"Abort message": "abort_message",
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +68,8 @@ class TombstoneCrashResult(pydantic.BaseModel):
|
||||
uid: int
|
||||
signal_info: SignalInfo
|
||||
cause: Optional[str] = None
|
||||
causes: Optional[List[dict]] = None
|
||||
abort_message: Optional[str] = None
|
||||
extra: Optional[str] = None
|
||||
|
||||
|
||||
|
||||
+30
-47
@@ -8,13 +8,13 @@ from zipfile import BadZipFile
|
||||
|
||||
import click
|
||||
|
||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||
from mvt.common.completion import (
|
||||
SUPPORTED_SHELLS,
|
||||
completion_instructions,
|
||||
generate_completion_script,
|
||||
install_completion_script,
|
||||
from mvt.common.cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||
load_cli_commands_option,
|
||||
register_cli_plugins,
|
||||
)
|
||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||
from mvt.common.help import (
|
||||
HELP_MSG_ANDROID_BACKUP_PASSWORD,
|
||||
HELP_MSG_CHECK_ADB_REMOVED,
|
||||
@@ -25,7 +25,6 @@ from mvt.common.help import (
|
||||
HELP_MSG_CHECK_IOCS,
|
||||
HELP_MSG_CHECK_INTRUSION_LOGS,
|
||||
HELP_MSG_DELAY_CHECKS,
|
||||
HELP_MSG_COMPLETION,
|
||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
||||
HELP_MSG_HASHES,
|
||||
@@ -82,6 +81,7 @@ def _load_custom_modules(load_module):
|
||||
# Main
|
||||
# ==============================================================================
|
||||
@click.group(invoke_without_command=False)
|
||||
@load_cli_commands_option
|
||||
@click.option(
|
||||
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
||||
)
|
||||
@@ -95,55 +95,20 @@ def cli(ctx, disable_update_check, disable_indicator_update_check):
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["disable_version_check"] = disable_update_check
|
||||
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
||||
if ctx.invoked_subcommand != "completion":
|
||||
logo(
|
||||
disable_version_check=disable_update_check,
|
||||
disable_indicator_check=disable_indicator_update_check,
|
||||
)
|
||||
logo(
|
||||
disable_version_check=disable_update_check,
|
||||
disable_indicator_check=disable_indicator_update_check,
|
||||
)
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: version
|
||||
# ==============================================================================
|
||||
@cli.command("version", help=HELP_MSG_VERSION)
|
||||
@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION)
|
||||
def version():
|
||||
return
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: completion
|
||||
# ==============================================================================
|
||||
@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION)
|
||||
@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS))
|
||||
@click.option(
|
||||
"--install",
|
||||
is_flag=True,
|
||||
help="Write completion files and update shell configuration.",
|
||||
)
|
||||
@click.pass_context
|
||||
def completion(ctx, shell, install):
|
||||
program_name = "mvt-android"
|
||||
|
||||
if shell is None:
|
||||
if install:
|
||||
raise click.UsageError("A shell is required when using --install.")
|
||||
click.echo(completion_instructions(program_name))
|
||||
return
|
||||
|
||||
root_cli = ctx.find_root().command
|
||||
|
||||
if install:
|
||||
script_path = install_completion_script(root_cli, program_name, shell)
|
||||
click.echo(f"Installed {shell} completion to {script_path}")
|
||||
if shell in ("bash", "zsh"):
|
||||
click.echo(f"Updated ~/.{shell}rc")
|
||||
else:
|
||||
click.echo("Fish loads completion files automatically.")
|
||||
return
|
||||
|
||||
click.echo(generate_completion_script(root_cli, program_name, shell))
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: check-adb (removed)
|
||||
# ==============================================================================
|
||||
@@ -505,3 +470,21 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
||||
def download_indicators():
|
||||
ioc_updates = IndicatorsUpdates()
|
||||
ioc_updates.update()
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Entry point of the mvt-android console script
|
||||
# ==============================================================================
|
||||
def main() -> None:
|
||||
"""Register the external commands and run the mvt-android CLI.
|
||||
|
||||
External commands are registered here rather than when this module is
|
||||
imported, so that importing MVT never runs third-party code and a plugin
|
||||
importing from MVT cannot re-enter a module that is still initializing.
|
||||
"""
|
||||
register_cli_plugins(
|
||||
cli,
|
||||
entry_point_group=ANDROID_CLI_PLUGIN_GROUP,
|
||||
environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
cli()
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import click
|
||||
|
||||
from mvt.common.cli_plugins import (
|
||||
MVT_CUSTOM_COMMANDS_ENV,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
load_cli_commands_option,
|
||||
register_cli_plugins,
|
||||
)
|
||||
from mvt.common.cmd_plugins import plugins
|
||||
from mvt.common.completion import completion
|
||||
from mvt.common.help import (
|
||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
||||
HELP_MSG_STIX2,
|
||||
HELP_MSG_VERSION,
|
||||
)
|
||||
from mvt.common.logo import logo
|
||||
from mvt.common.updates import IndicatorsUpdates
|
||||
from mvt.common.utils import init_logging
|
||||
|
||||
init_logging()
|
||||
|
||||
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Main
|
||||
# ==============================================================================
|
||||
@click.group(invoke_without_command=True)
|
||||
@load_cli_commands_option
|
||||
@click.option(
|
||||
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
||||
)
|
||||
@click.option(
|
||||
"--disable-indicator-update-check",
|
||||
is_flag=True,
|
||||
help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||
)
|
||||
@click.pass_context
|
||||
def cli(ctx, disable_update_check, disable_indicator_update_check):
|
||||
"""Mobile Verification Toolkit.
|
||||
|
||||
mvt-ios and mvt-android run the forensic analysis of an acquisition: each
|
||||
provides the check-* commands of its platform. This command hosts what
|
||||
belongs to neither platform; run it without a command to see the installed
|
||||
version and the list of what it offers.
|
||||
"""
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["disable_version_check"] = disable_update_check
|
||||
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
||||
if ctx.invoked_subcommand != "completion":
|
||||
logo(
|
||||
disable_version_check=disable_update_check,
|
||||
disable_indicator_check=disable_indicator_update_check,
|
||||
)
|
||||
if ctx.invoked_subcommand is None:
|
||||
click.echo(ctx.get_help())
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: download-iocs
|
||||
# ==============================================================================
|
||||
@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
|
||||
def download_iocs():
|
||||
ioc_updates = IndicatorsUpdates()
|
||||
ioc_updates.update()
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: completion
|
||||
# ==============================================================================
|
||||
cli.add_command(completion)
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: version
|
||||
# ==============================================================================
|
||||
@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION)
|
||||
def version():
|
||||
return
|
||||
|
||||
|
||||
# The plugins command is registered as a built-in command, before any external
|
||||
# command, so that an installed package can never replace it.
|
||||
cli.add_command(plugins)
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Entry point of the mvt console script
|
||||
# ==============================================================================
|
||||
def main() -> None:
|
||||
"""Register the external commands and run the mvt CLI.
|
||||
|
||||
External commands are registered here rather than when this module is
|
||||
imported, so that importing MVT never runs third-party code and a plugin
|
||||
importing from MVT cannot re-enter a module that is still initializing.
|
||||
"""
|
||||
register_cli_plugins(
|
||||
cli,
|
||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
cli()
|
||||
@@ -0,0 +1,310 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Iterable
|
||||
|
||||
import click
|
||||
|
||||
IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins"
|
||||
ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins"
|
||||
# Commands in this group are registered on the platform-neutral mvt command only.
|
||||
NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins"
|
||||
MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS"
|
||||
MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS"
|
||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS"
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CustomCommandLoadError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BrokenPluginCommand(click.Command):
|
||||
"""A placeholder for an installed or configured command that failed to load."""
|
||||
|
||||
def __init__(self, name: str, source: str, exception: BaseException):
|
||||
super().__init__(
|
||||
name,
|
||||
help=(
|
||||
f"Unable to load external command from {source}.\n\n"
|
||||
f"{type(exception).__name__}: {exception}"
|
||||
),
|
||||
short_help="Warning: external command could not be loaded.",
|
||||
)
|
||||
self.source = source
|
||||
self.exception = exception
|
||||
|
||||
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
||||
return args
|
||||
|
||||
def invoke(self, ctx: click.Context) -> None:
|
||||
raise click.ClickException(
|
||||
f"Unable to load external command '{self.name}' from {self.source}: "
|
||||
f"{type(self.exception).__name__}: {self.exception}"
|
||||
)
|
||||
|
||||
|
||||
def _module_name_for_path(path: Path) -> str:
|
||||
digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16]
|
||||
return f"_mvt_custom_command_{path.stem}_{digest}"
|
||||
|
||||
|
||||
def _iter_command_files(path: Path) -> Iterable[Path]:
|
||||
if path.is_file():
|
||||
if path.suffix != ".py":
|
||||
raise CustomCommandLoadError(
|
||||
f"Custom command file is not a Python file: {path}"
|
||||
)
|
||||
yield path
|
||||
return
|
||||
|
||||
if path.is_dir():
|
||||
for child in sorted(path.iterdir()):
|
||||
if child.name.startswith(".") or child.name == "__init__.py":
|
||||
continue
|
||||
if child.is_file() and child.suffix == ".py":
|
||||
yield child
|
||||
return
|
||||
|
||||
raise CustomCommandLoadError(f"Custom command path does not exist: {path}")
|
||||
|
||||
|
||||
def _load_python_file(path: Path) -> ModuleType:
|
||||
module_name = _module_name_for_path(path)
|
||||
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise CustomCommandLoadError(f"Unable to load custom command file: {path}")
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except (Exception, SystemExit) as exc:
|
||||
raise CustomCommandLoadError(
|
||||
f"Unable to import custom command {path}: {exc}"
|
||||
) from exc
|
||||
|
||||
return module
|
||||
|
||||
|
||||
def load_cli_command_file(path: Path) -> click.Command:
|
||||
module = _load_python_file(path)
|
||||
command = getattr(module, "cli", None)
|
||||
if not isinstance(command, click.Command):
|
||||
raise CustomCommandLoadError(
|
||||
f"Custom command {path} must export a Click command or group named 'cli'"
|
||||
)
|
||||
if command.name is None:
|
||||
raise CustomCommandLoadError(f"Custom command {path} has no command name")
|
||||
return command
|
||||
|
||||
|
||||
def _register_command(
|
||||
group: click.Group,
|
||||
command: click.Command,
|
||||
*,
|
||||
name: str,
|
||||
source: str,
|
||||
collision_is_error: bool,
|
||||
) -> bool:
|
||||
if name in group.commands:
|
||||
registered_sources = getattr(group, "_mvt_external_command_sources", {})
|
||||
if registered_sources.get(name) == source:
|
||||
return False
|
||||
message = (
|
||||
f"Unable to register external command '{name}' from {source}: "
|
||||
"the command name is already registered"
|
||||
)
|
||||
if collision_is_error:
|
||||
raise CustomCommandLoadError(message)
|
||||
log.warning(message)
|
||||
return False
|
||||
|
||||
group.add_command(command, name=name)
|
||||
registered_sources = getattr(group, "_mvt_external_command_sources", {})
|
||||
registered_sources[name] = source
|
||||
setattr(group, "_mvt_external_command_sources", registered_sources)
|
||||
return True
|
||||
|
||||
|
||||
def register_cli_commands_from_path(
|
||||
group: click.Group,
|
||||
path: str | Path,
|
||||
*,
|
||||
collision_is_error: bool = False,
|
||||
failures_are_errors: bool = False,
|
||||
) -> list[str]:
|
||||
resolved_path = Path(path).expanduser().resolve()
|
||||
registered: list[str] = []
|
||||
|
||||
try:
|
||||
command_files = list(_iter_command_files(resolved_path))
|
||||
except CustomCommandLoadError as exc:
|
||||
if failures_are_errors:
|
||||
raise
|
||||
name = resolved_path.stem.replace("_", "-") or "custom-command"
|
||||
if _register_command(
|
||||
group,
|
||||
BrokenPluginCommand(name, str(resolved_path), exc),
|
||||
name=name,
|
||||
source=str(resolved_path),
|
||||
collision_is_error=collision_is_error,
|
||||
):
|
||||
registered.append(name)
|
||||
return registered
|
||||
|
||||
for command_file in command_files:
|
||||
try:
|
||||
command = load_cli_command_file(command_file)
|
||||
except CustomCommandLoadError as exc:
|
||||
if failures_are_errors:
|
||||
raise
|
||||
name = command_file.stem.replace("_", "-")
|
||||
command = BrokenPluginCommand(name, str(command_file), exc)
|
||||
|
||||
command_name = command.name
|
||||
if command_name is None:
|
||||
raise CustomCommandLoadError(
|
||||
f"Custom command {command_file} has no command name"
|
||||
)
|
||||
if _register_command(
|
||||
group,
|
||||
command,
|
||||
name=command_name,
|
||||
source=str(command_file),
|
||||
collision_is_error=collision_is_error,
|
||||
):
|
||||
registered.append(command_name)
|
||||
|
||||
return registered
|
||||
|
||||
|
||||
def _entry_point_source(entry_point: importlib.metadata.EntryPoint) -> str:
|
||||
try:
|
||||
distribution = getattr(entry_point, "dist", None)
|
||||
if distribution is None:
|
||||
return f"entry point {entry_point.value}"
|
||||
|
||||
name = distribution.metadata.get("Name", "unknown distribution")
|
||||
version = getattr(distribution, "version", None)
|
||||
if version:
|
||||
return f"{name} {version} ({entry_point.value})"
|
||||
return f"{name} ({entry_point.value})"
|
||||
except Exception:
|
||||
return f"entry point {entry_point.value}"
|
||||
|
||||
|
||||
def register_installed_cli_commands(
|
||||
group: click.Group,
|
||||
entry_point_group: str,
|
||||
) -> list[str]:
|
||||
try:
|
||||
entry_points = importlib.metadata.entry_points(group=entry_point_group)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Unable to discover external commands in entry-point group %s: %s",
|
||||
entry_point_group,
|
||||
exc,
|
||||
)
|
||||
return []
|
||||
ordered_entry_points = sorted(
|
||||
entry_points,
|
||||
key=lambda entry_point: (
|
||||
entry_point.name,
|
||||
_entry_point_source(entry_point),
|
||||
entry_point.value,
|
||||
),
|
||||
)
|
||||
registered: list[str] = []
|
||||
|
||||
for entry_point in ordered_entry_points:
|
||||
source = _entry_point_source(entry_point)
|
||||
try:
|
||||
command = entry_point.load()
|
||||
if not isinstance(command, click.Command):
|
||||
raise TypeError(
|
||||
f"entry point must resolve to a Click command or group, "
|
||||
f"not {type(command).__name__}"
|
||||
)
|
||||
except (Exception, SystemExit) as exc:
|
||||
command = BrokenPluginCommand(entry_point.name, source, exc)
|
||||
|
||||
if _register_command(
|
||||
group,
|
||||
command,
|
||||
name=entry_point.name,
|
||||
source=source,
|
||||
collision_is_error=False,
|
||||
):
|
||||
registered.append(entry_point.name)
|
||||
|
||||
return registered
|
||||
|
||||
|
||||
def register_cli_plugins(
|
||||
group: click.Group,
|
||||
*,
|
||||
entry_point_group: str,
|
||||
environment_variable: str,
|
||||
) -> None:
|
||||
"""Register the external commands of one CLI on its group.
|
||||
|
||||
Each CLI has one entry-point group and one environment variable of its
|
||||
own, so a command package chooses the CLIs its commands are added to.
|
||||
|
||||
:param group: CLI group to register the external commands on.
|
||||
:param entry_point_group: Entry-point group of the CLI.
|
||||
:param environment_variable: Name of the environment variable holding a
|
||||
path to load commands from.
|
||||
"""
|
||||
environment_path = os.environ.get(environment_variable)
|
||||
if environment_path:
|
||||
register_cli_commands_from_path(group, environment_path)
|
||||
register_installed_cli_commands(group, entry_point_group)
|
||||
|
||||
|
||||
def _load_command_option_callback(
|
||||
ctx: click.Context,
|
||||
param: click.Parameter,
|
||||
paths: tuple[str, ...],
|
||||
) -> tuple[str, ...]:
|
||||
if not isinstance(ctx.command, click.Group):
|
||||
raise click.ClickException("--load-command requires a Click command group")
|
||||
|
||||
for path in paths:
|
||||
try:
|
||||
register_cli_commands_from_path(
|
||||
ctx.command,
|
||||
path,
|
||||
collision_is_error=True,
|
||||
failures_are_errors=True,
|
||||
)
|
||||
except CustomCommandLoadError as exc:
|
||||
raise click.BadParameter(str(exc), ctx=ctx, param=param) from exc
|
||||
return paths
|
||||
|
||||
|
||||
load_cli_commands_option = click.option(
|
||||
"--load-command",
|
||||
"load_commands",
|
||||
type=click.Path(exists=True),
|
||||
multiple=True,
|
||||
expose_value=False,
|
||||
is_eager=True,
|
||||
callback=_load_command_option_callback,
|
||||
help=(
|
||||
"Load a custom CLI command from a Python file or folder "
|
||||
"(can be invoked multiple times)"
|
||||
),
|
||||
)
|
||||
@@ -9,6 +9,7 @@ from typing import Optional
|
||||
|
||||
from mvt.common.command import Command
|
||||
from mvt.common.module import MVTModule
|
||||
from mvt.common.module_loader import get_module_logger
|
||||
from mvt.common.utils import exec_or_profile
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -76,7 +77,7 @@ class CmdCheckIOCS(Command):
|
||||
)
|
||||
|
||||
m = iocs_module.from_json(
|
||||
file_path, log=logging.getLogger(iocs_module.__module__)
|
||||
file_path, log=get_module_logger(iocs_module)
|
||||
)
|
||||
if not m:
|
||||
log.warning("No result from this module, skipping it")
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import importlib.metadata
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from .cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
from .config import settings
|
||||
from .help import (
|
||||
HELP_MSG_PLUGINS,
|
||||
HELP_MSG_PLUGINS_CHECK_UPDATES,
|
||||
HELP_MSG_PLUGINS_LIST,
|
||||
)
|
||||
from .module import MVTModule
|
||||
from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url
|
||||
from .updates import (
|
||||
SHORT_COMMIT_LENGTH,
|
||||
PluginUpdates,
|
||||
installed_plugin_distributions,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CLI_PLUGIN_GROUPS = (
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
||||
|
||||
|
||||
def _entry_points(group: str) -> list[importlib.metadata.EntryPoint]:
|
||||
try:
|
||||
return list(importlib.metadata.entry_points(group=group))
|
||||
except Exception as exc:
|
||||
log.warning("Unable to discover the entry points in group %s: %s", group, exc)
|
||||
return []
|
||||
|
||||
|
||||
def _entry_point_distribution(
|
||||
entry_point: importlib.metadata.EntryPoint,
|
||||
) -> Optional[str]:
|
||||
dist = getattr(entry_point, "dist", None)
|
||||
if dist is None:
|
||||
return None
|
||||
try:
|
||||
return dist.name
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _distribution_version(dist: importlib.metadata.Distribution) -> str:
|
||||
try:
|
||||
return dist.version or "unknown"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _distribution_origin(dist: importlib.metadata.Distribution) -> str:
|
||||
"""Describe where a plugin package was installed from."""
|
||||
direct_url = distribution_direct_url(dist)
|
||||
if direct_url is None:
|
||||
return "pypi"
|
||||
|
||||
vcs_info = direct_url.get("vcs_info")
|
||||
if isinstance(vcs_info, dict):
|
||||
commit = vcs_info.get("commit_id") or ""
|
||||
if commit:
|
||||
return f"git+{commit[:SHORT_COMMIT_LENGTH]}"
|
||||
return "git"
|
||||
|
||||
return "local"
|
||||
|
||||
|
||||
def _contributed_modules(
|
||||
entry_points: list[importlib.metadata.EntryPoint], distribution: str
|
||||
) -> str:
|
||||
"""Count the forensic modules a plugin package contributes.
|
||||
|
||||
Entry points are resolved the way MVT resolves them when it loads
|
||||
modules, but a broken entry point is reported instead of raising: listing
|
||||
the installed plugins must work even when one of them is faulty.
|
||||
"""
|
||||
count = 0
|
||||
broken = False
|
||||
|
||||
for entry_point in entry_points:
|
||||
if _entry_point_distribution(entry_point) != distribution:
|
||||
continue
|
||||
try:
|
||||
loaded = entry_point.load()
|
||||
if callable(loaded) and not isinstance(loaded, type):
|
||||
loaded = loaded()
|
||||
count += sum(
|
||||
1
|
||||
for module in loaded
|
||||
if isinstance(module, type) and issubclass(module, MVTModule)
|
||||
)
|
||||
except (Exception, SystemExit) as exc:
|
||||
log.debug(
|
||||
"Unable to load the modules of entry point %s (%s): %s",
|
||||
entry_point.name,
|
||||
entry_point.value,
|
||||
exc,
|
||||
)
|
||||
broken = True
|
||||
|
||||
if broken:
|
||||
return f"{count} (error)" if count else "error"
|
||||
|
||||
return str(count)
|
||||
|
||||
|
||||
def _contributed_commands(
|
||||
entry_points: list[importlib.metadata.EntryPoint], distribution: str
|
||||
) -> str:
|
||||
names = {
|
||||
entry_point.name
|
||||
for entry_point in entry_points
|
||||
if _entry_point_distribution(entry_point) == distribution
|
||||
}
|
||||
|
||||
return ", ".join(sorted(names)) if names else "-"
|
||||
|
||||
|
||||
@click.group("plugins", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS)
|
||||
def plugins() -> None:
|
||||
pass
|
||||
|
||||
|
||||
@plugins.command("list", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS_LIST)
|
||||
def list_plugins() -> None:
|
||||
distributions = installed_plugin_distributions()
|
||||
if not distributions:
|
||||
click.echo("No MVT plugins are installed.")
|
||||
return
|
||||
|
||||
module_entry_points = _entry_points(MODULES_ENTRY_POINT_GROUP)
|
||||
command_entry_points = []
|
||||
for group in CLI_PLUGIN_GROUPS:
|
||||
command_entry_points.extend(_entry_points(group))
|
||||
|
||||
table = Table(title="Installed MVT plugins")
|
||||
table.add_column("Name", style="bold")
|
||||
table.add_column("Version")
|
||||
table.add_column("Origin")
|
||||
table.add_column("Modules", justify="right")
|
||||
table.add_column("Commands")
|
||||
|
||||
for dist in distributions:
|
||||
name = dist.name
|
||||
table.add_row(
|
||||
name,
|
||||
_distribution_version(dist),
|
||||
_distribution_origin(dist),
|
||||
_contributed_modules(module_entry_points, name),
|
||||
_contributed_commands(command_entry_points, name),
|
||||
)
|
||||
|
||||
Console().print(table)
|
||||
|
||||
|
||||
@plugins.command(
|
||||
"check-updates",
|
||||
context_settings=CONTEXT_SETTINGS,
|
||||
help=HELP_MSG_PLUGINS_CHECK_UPDATES,
|
||||
short_help="Check the installed plugins for updates",
|
||||
)
|
||||
def check_plugin_updates() -> None:
|
||||
if not settings.NETWORK_ACCESS_ALLOWED:
|
||||
click.echo(
|
||||
"Network access is disabled, cannot check for plugin updates. "
|
||||
"Enable NETWORK_ACCESS_ALLOWED in the MVT configuration to check."
|
||||
)
|
||||
return
|
||||
|
||||
if not installed_plugin_distributions():
|
||||
click.echo("No MVT plugins are installed.")
|
||||
return
|
||||
|
||||
findings = PluginUpdates().check()
|
||||
if not findings:
|
||||
click.echo("All plugins are up to date.")
|
||||
return
|
||||
|
||||
click.echo("Plugin updates available:")
|
||||
for finding in findings:
|
||||
click.echo(f" {finding['name']} {finding['installed']} → {finding['latest']}")
|
||||
click.echo(f" Upgrade with: {finding['upgrade_command']}")
|
||||
|
||||
click.echo(
|
||||
"\nMVT does not install plugin updates. Run the command above when you "
|
||||
"decide to upgrade."
|
||||
)
|
||||
+130
-20
@@ -19,7 +19,12 @@ from .alerts import AlertLevel, AlertStore
|
||||
from .config import settings
|
||||
from .indicators import Indicators
|
||||
from .module import EncryptedBackupError, MVTModule, run_module, save_timeline
|
||||
from .module_loader import module_supports_command
|
||||
from .module_loader import (
|
||||
ModuleOrigin,
|
||||
get_module_logger,
|
||||
get_module_origin,
|
||||
module_supports_command,
|
||||
)
|
||||
from .module_types import ModuleTimeline, URLResult
|
||||
from .utils import (
|
||||
CustomJSONEncoder,
|
||||
@@ -210,10 +215,34 @@ class Command:
|
||||
for file in generate_hashes_from_path(self.target_path, self.log):
|
||||
self.hash_values.append(file)
|
||||
|
||||
@staticmethod
|
||||
def _modules_by_origin(
|
||||
modules: list[type[MVTModule]],
|
||||
) -> dict[ModuleOrigin, list[str]]:
|
||||
origins: dict[ModuleOrigin, list[str]] = {}
|
||||
for module in modules:
|
||||
origins.setdefault(get_module_origin(module), []).append(module.__name__)
|
||||
return origins
|
||||
|
||||
def list_modules(self) -> None:
|
||||
self.log.info("Following is the list of available %s modules:", self.name)
|
||||
for module in self._available_modules():
|
||||
self.log.info(" - %s", module.__name__)
|
||||
for origin, module_names in self._modules_by_origin(
|
||||
self._available_modules()
|
||||
).items():
|
||||
self.log.info(
|
||||
" - Modules from %s: %s", origin.label, ", ".join(module_names)
|
||||
)
|
||||
|
||||
def _log_loaded_modules(self, modules: list[type[MVTModule]]) -> None:
|
||||
"""Record the loaded modules and their origins for auditability."""
|
||||
for origin, module_names in self._modules_by_origin(modules).items():
|
||||
self.log.info(
|
||||
"Loaded %d %s modules from %s: %s",
|
||||
len(module_names),
|
||||
self.name,
|
||||
origin.label,
|
||||
", ".join(module_names),
|
||||
)
|
||||
|
||||
def _available_modules(self) -> list[type[MVTModule]]:
|
||||
modules = list(self.modules)
|
||||
@@ -288,6 +317,81 @@ class Command:
|
||||
console.print("")
|
||||
console.print(panel)
|
||||
|
||||
def _skipped_modules(
|
||||
self,
|
||||
required: list[type[MVTModule]],
|
||||
module_indexes: dict[type[MVTModule], int],
|
||||
) -> dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]]:
|
||||
"""Return the modules to drop because a dependency is unavailable.
|
||||
|
||||
A module declaring a dependency this command cannot provide is unable
|
||||
to run, and so is every module depending on it. Dropping only those
|
||||
keeps a single wrong declaration - in a module scoped to several
|
||||
commands, for example - from silencing an entire analysis.
|
||||
|
||||
The returned mapping gives, for each skipped module, the module which
|
||||
is missing a dependency and the dependency it is missing.
|
||||
"""
|
||||
skipped: dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]] = {}
|
||||
# Skipping the one module a run was asked for leaves nothing to run,
|
||||
# which the caller reports instead.
|
||||
remainder = (
|
||||
"" if self.module_name else " The rest of the analysis will still run."
|
||||
)
|
||||
|
||||
# Skipping one module can skip the modules depending on it, which the
|
||||
# pass over the module list may already have gone past, so repeat the
|
||||
# pass until nothing changes.
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for module in required:
|
||||
if module in skipped:
|
||||
continue
|
||||
|
||||
for dependency in module.dependencies:
|
||||
if dependency not in module_indexes:
|
||||
skipped[module] = (module, dependency)
|
||||
changed = True
|
||||
self.log.warning(
|
||||
"Module %s will be SKIPPED: it depends on module "
|
||||
"%s, which is not available in this command.%s",
|
||||
module.__name__,
|
||||
dependency.__name__,
|
||||
remainder,
|
||||
)
|
||||
break
|
||||
|
||||
if dependency in skipped:
|
||||
root, missing = skipped[dependency]
|
||||
skipped[module] = (root, missing)
|
||||
changed = True
|
||||
if dependency is root:
|
||||
self.log.warning(
|
||||
"Module %s will be SKIPPED: it depends on "
|
||||
"module %s, itself skipped for depending on "
|
||||
"unavailable module %s.%s",
|
||||
module.__name__,
|
||||
dependency.__name__,
|
||||
missing.__name__,
|
||||
remainder,
|
||||
)
|
||||
else:
|
||||
self.log.warning(
|
||||
"Module %s will be SKIPPED: it depends on "
|
||||
"skipped module %s, in a chain starting at "
|
||||
"module %s, which depends on unavailable "
|
||||
"module %s.%s",
|
||||
module.__name__,
|
||||
dependency.__name__,
|
||||
root.__name__,
|
||||
missing.__name__,
|
||||
remainder,
|
||||
)
|
||||
break
|
||||
|
||||
return skipped
|
||||
|
||||
def _ordered_modules(self) -> Optional[list[type[MVTModule]]]:
|
||||
"""Return enabled modules in stable topological order."""
|
||||
modules = self._available_modules()
|
||||
@@ -300,30 +404,34 @@ class Command:
|
||||
else:
|
||||
selected = [module for module in modules if module.enabled]
|
||||
|
||||
required = set(selected)
|
||||
required: set[type[MVTModule]] = set()
|
||||
pending = list(selected)
|
||||
while pending:
|
||||
module = pending.pop()
|
||||
if module in required:
|
||||
continue
|
||||
required.add(module)
|
||||
for dependency in module.dependencies:
|
||||
if dependency not in module_indexes:
|
||||
self.log.warning(
|
||||
"Module %s depends on unavailable module %s. "
|
||||
"No modules will be run.",
|
||||
module.__name__,
|
||||
dependency.__name__,
|
||||
)
|
||||
return None
|
||||
if dependency not in required:
|
||||
required.add(dependency)
|
||||
# Unavailable dependencies are reported by _skipped_modules().
|
||||
if dependency in module_indexes:
|
||||
pending.append(dependency)
|
||||
|
||||
ordered_required = sorted(required, key=lambda module: module_indexes[module])
|
||||
skipped = self._skipped_modules(ordered_required, module_indexes)
|
||||
runnable = [module for module in ordered_required if module not in skipped]
|
||||
if skipped and not runnable:
|
||||
self.log.warning(
|
||||
"Every selected module was skipped for an unavailable "
|
||||
"dependency. No modules will be run."
|
||||
)
|
||||
|
||||
dependents: dict[type[MVTModule], list[type[MVTModule]]] = {
|
||||
module: [] for module in required
|
||||
module: [] for module in runnable
|
||||
}
|
||||
indegree = {module: 0 for module in required}
|
||||
for module in required:
|
||||
indegree = {module: 0 for module in runnable}
|
||||
for module in runnable:
|
||||
for dependency in module.dependencies:
|
||||
if dependency not in required:
|
||||
if dependency not in indegree:
|
||||
continue
|
||||
dependents[dependency].append(module)
|
||||
indegree[module] += 1
|
||||
@@ -342,7 +450,7 @@ class Command:
|
||||
if indegree[dependent] == 0:
|
||||
heappush(ready, (module_indexes[dependent], dependent))
|
||||
|
||||
if len(ordered) != len(required):
|
||||
if len(ordered) != len(runnable):
|
||||
cyclic_modules = sorted(
|
||||
(module.__name__ for module, count in indegree.items() if count > 0)
|
||||
)
|
||||
@@ -360,6 +468,8 @@ class Command:
|
||||
if ordered_modules is None:
|
||||
return
|
||||
|
||||
self._log_loaded_modules(ordered_modules)
|
||||
|
||||
try:
|
||||
self.init()
|
||||
except NotImplementedError:
|
||||
@@ -368,7 +478,7 @@ class Command:
|
||||
executed_by_type: dict[type[MVTModule], MVTModule] = {}
|
||||
for module in ordered_modules:
|
||||
|
||||
module_logger = logging.getLogger(module.__module__)
|
||||
module_logger = get_module_logger(module)
|
||||
|
||||
m = module(
|
||||
target_path=self.target_path,
|
||||
|
||||
@@ -9,34 +9,83 @@ import shlex
|
||||
import click
|
||||
from click.shell_completion import get_completion_class
|
||||
|
||||
from .help import HELP_MSG_COMPLETION
|
||||
|
||||
|
||||
SUPPORTED_SHELLS = ("bash", "zsh", "fish")
|
||||
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
||||
|
||||
COMPLETION_INSTRUCTIONS = """Shell completion for mvt, mvt-ios and mvt-android
|
||||
|
||||
def completion_instructions(program_name: str) -> str:
|
||||
return f"""Shell completion for {program_name}
|
||||
|
||||
Print a completion script:
|
||||
{program_name} completion bash > ~/.{program_name}-complete.bash
|
||||
{program_name} completion zsh > ~/.{program_name}-complete.zsh
|
||||
mkdir -p ~/.config/fish/completions
|
||||
{program_name} completion fish > ~/.config/fish/completions/{program_name}.fish
|
||||
Print one completion script covering the three commands:
|
||||
mvt completion bash > ~/.mvt-complete.bash
|
||||
mvt completion zsh > ~/.mvt-complete.zsh
|
||||
mkdir -p ~/.config/fish/conf.d
|
||||
mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish
|
||||
|
||||
Load the generated Bash script from ~/.bashrc:
|
||||
[ -f ~/.{program_name}-complete.bash ] && . ~/.{program_name}-complete.bash
|
||||
[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash
|
||||
|
||||
Load the generated Zsh script from ~/.zshrc:
|
||||
[ -f ~/.{program_name}-complete.zsh ] && . ~/.{program_name}-complete.zsh
|
||||
[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh
|
||||
|
||||
Fish loads completion files from ~/.config/fish/completions automatically.
|
||||
Fish loads the files in ~/.config/fish/conf.d automatically.
|
||||
|
||||
To write these files and update Bash/Zsh shell configuration automatically:
|
||||
{program_name} completion bash --install
|
||||
{program_name} completion zsh --install
|
||||
{program_name} completion fish --install
|
||||
To write these files and update the Bash/Zsh shell configuration automatically:
|
||||
mvt completion bash --install
|
||||
mvt completion zsh --install
|
||||
mvt completion fish --install
|
||||
"""
|
||||
|
||||
|
||||
def _mvt_programs() -> list[tuple[str, click.Command]]:
|
||||
"""Return the console script name and CLI group of every MVT program.
|
||||
|
||||
The three CLIs are imported here rather than at module level: mvt.cli
|
||||
imports this module while it is being defined, and generating a completion
|
||||
script should not make the start-up of `mvt` import the platform CLIs.
|
||||
"""
|
||||
from mvt.android.cli import cli as android_cli
|
||||
from mvt.cli import cli as mvt_cli
|
||||
from mvt.ios.cli import cli as ios_cli
|
||||
|
||||
return [("mvt", mvt_cli), ("mvt-ios", ios_cli), ("mvt-android", android_cli)]
|
||||
|
||||
|
||||
@click.command(
|
||||
"completion",
|
||||
context_settings=CONTEXT_SETTINGS,
|
||||
help=HELP_MSG_COMPLETION,
|
||||
short_help="Generate or install shell completion",
|
||||
)
|
||||
@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS))
|
||||
@click.option(
|
||||
"--install",
|
||||
is_flag=True,
|
||||
help="Write completion files and update shell configuration.",
|
||||
)
|
||||
def completion(shell, install):
|
||||
if shell is None:
|
||||
if install:
|
||||
raise click.UsageError("A shell is required when using --install.")
|
||||
click.echo(COMPLETION_INSTRUCTIONS)
|
||||
return
|
||||
|
||||
if install:
|
||||
script_path = install_completion_script(shell)
|
||||
click.echo(
|
||||
f"Installed {shell} completion for mvt, mvt-ios and mvt-android "
|
||||
f"to {script_path}"
|
||||
)
|
||||
if shell in ("bash", "zsh"):
|
||||
click.echo(f"Updated ~/.{shell}rc")
|
||||
else:
|
||||
click.echo("Fish loads the files in ~/.config/fish/conf.d automatically.")
|
||||
return
|
||||
|
||||
click.echo(generate_mvt_completion_script(shell))
|
||||
|
||||
|
||||
def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str:
|
||||
completion_class = get_completion_class(shell)
|
||||
if completion_class is None:
|
||||
@@ -46,41 +95,49 @@ def generate_completion_script(cli: click.Command, program_name: str, shell: str
|
||||
return completion_class(cli, {}, program_name, complete_var).source()
|
||||
|
||||
|
||||
def install_completion_script(
|
||||
cli: click.Command,
|
||||
program_name: str,
|
||||
shell: str,
|
||||
) -> Path:
|
||||
script = generate_completion_script(cli, program_name, shell)
|
||||
script_path = _completion_script_path(program_name, shell)
|
||||
def generate_mvt_completion_script(shell: str) -> str:
|
||||
"""Return one script completing every MVT command.
|
||||
|
||||
Click names the completion function of each program after the program, so
|
||||
the scripts of the three commands can simply be concatenated.
|
||||
"""
|
||||
scripts = [
|
||||
generate_completion_script(cli, program_name, shell).strip("\n")
|
||||
for program_name, cli in _mvt_programs()
|
||||
]
|
||||
return "\n\n".join(scripts)
|
||||
|
||||
|
||||
def install_completion_script(shell: str) -> Path:
|
||||
script = generate_mvt_completion_script(shell)
|
||||
script_path = _completion_script_path(shell)
|
||||
script_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
script_path.write_text(script, encoding="utf-8")
|
||||
script_path.write_text(f"{script}\n", encoding="utf-8")
|
||||
|
||||
if shell in ("bash", "zsh"):
|
||||
_install_shell_source_line(program_name, shell, script_path)
|
||||
_install_shell_source_line(shell, script_path)
|
||||
|
||||
return script_path
|
||||
|
||||
|
||||
def _completion_script_path(program_name: str, shell: str) -> Path:
|
||||
def _completion_script_path(shell: str) -> Path:
|
||||
home = Path.home()
|
||||
|
||||
if shell == "fish":
|
||||
return home / ".config" / "fish" / "completions" / f"{program_name}.fish"
|
||||
# conf.d is sourced when the shell starts, unlike the completions
|
||||
# folder, whose files fish loads on demand by command name.
|
||||
return home / ".config" / "fish" / "conf.d" / "mvt-completion.fish"
|
||||
|
||||
return home / f".{program_name}-complete.{shell}"
|
||||
return home / f".mvt-complete.{shell}"
|
||||
|
||||
|
||||
def _install_shell_source_line(program_name: str, shell: str, script_path: Path) -> None:
|
||||
def _install_shell_source_line(shell: str, script_path: Path) -> None:
|
||||
shell_config_path = Path.home() / f".{shell}rc"
|
||||
source_line = (
|
||||
f"[ -f {shlex.quote(str(script_path))} ] && "
|
||||
f". {shlex.quote(str(script_path))}"
|
||||
)
|
||||
block = (
|
||||
f"# MVT shell completion for {program_name}\n"
|
||||
f"{source_line}\n"
|
||||
)
|
||||
block = f"# MVT shell completion\n{source_line}\n"
|
||||
|
||||
if shell_config_path.exists():
|
||||
shell_config = shell_config_path.read_text(encoding="utf-8")
|
||||
|
||||
@@ -8,7 +8,7 @@ HELP_MSG_VERSION = "Show the currently installed version of MVT"
|
||||
HELP_MSG_OUTPUT = "Specify a path to a folder where you want to store JSON results"
|
||||
HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)"
|
||||
HELP_MSG_FAST = "Avoid running time/resource consuming features"
|
||||
HELP_MSG_LIST_MODULES = "Print list of available modules and exit"
|
||||
HELP_MSG_LIST_MODULES = "Print list of available modules and their source, then exit"
|
||||
HELP_MSG_MODULE = "Name of a single module you would like to run instead of all"
|
||||
HELP_MSG_LOAD_MODULE = (
|
||||
"Load custom MVT module(s) from a Python file or folder "
|
||||
@@ -21,7 +21,14 @@ HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators"
|
||||
HELP_MSG_STIX2 = "Download public STIX2 indicators"
|
||||
HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check"
|
||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check"
|
||||
HELP_MSG_COMPLETION = "Generate or install shell completion"
|
||||
HELP_MSG_COMPLETION = (
|
||||
"Generate or install shell completion for mvt, mvt-ios and mvt-android"
|
||||
)
|
||||
HELP_MSG_PLUGINS = "Inspect the installed MVT plugin packages"
|
||||
HELP_MSG_PLUGINS_LIST = "List the installed plugins and what they contribute to MVT"
|
||||
HELP_MSG_PLUGINS_CHECK_UPDATES = (
|
||||
"Check the installed plugins for updates without installing them"
|
||||
)
|
||||
|
||||
# IOS Specific
|
||||
HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup"
|
||||
|
||||
+125
-62
@@ -8,10 +8,125 @@ import logging
|
||||
import requests
|
||||
from rich import print as rich_print
|
||||
|
||||
from .updates import IndicatorsUpdates, MVTUpdates
|
||||
from .config import settings
|
||||
from .updates import (
|
||||
IndicatorsUpdates,
|
||||
MVTUpdates,
|
||||
PluginUpdates,
|
||||
installed_plugin_distributions,
|
||||
)
|
||||
from .version import MVT_VERSION
|
||||
|
||||
|
||||
def _check_version_updates(log: logging.Logger) -> None:
|
||||
try:
|
||||
mvt_updates = MVTUpdates()
|
||||
latest_version = mvt_updates.check()
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
rich_print(
|
||||
"\t[bold]Note: Could not check for MVT updates.[/bold] "
|
||||
"You may be working offline. Please update MVT regularly."
|
||||
)
|
||||
except Exception as e:
|
||||
log.error("Error encountered when trying to check latest MVT version: %s", e)
|
||||
else:
|
||||
if latest_version:
|
||||
rich_print(
|
||||
f"\t[bold]Version {latest_version} is available! "
|
||||
"Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
|
||||
)
|
||||
|
||||
|
||||
def _check_indicator_updates(log: logging.Logger) -> None:
|
||||
ioc_updates = IndicatorsUpdates()
|
||||
|
||||
# Before proceeding, we check if we have downloaded an indicators index.
|
||||
# If not, there's no point in proceeding with the updates check.
|
||||
if ioc_updates.get_latest_update() == 0:
|
||||
rich_print(
|
||||
"\t[bold]You have not yet downloaded any indicators, check "
|
||||
"the `download-iocs` command![/bold]"
|
||||
)
|
||||
return
|
||||
|
||||
# We only perform this check at a fixed frequency, in order to not
|
||||
# overburden the user with too many lookups if the command is being run
|
||||
# multiple times.
|
||||
should_check, hours = ioc_updates.should_check()
|
||||
if not should_check:
|
||||
rich_print(
|
||||
f"\tIndicators updates checked recently, next automatic check "
|
||||
f"in {int(hours)} hours"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
ioc_to_update = ioc_updates.check()
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
rich_print(
|
||||
"\t[bold]Note: Could not check for indicator updates.[/bold] "
|
||||
"You may be working offline. Please update MVT indicators regularly."
|
||||
)
|
||||
except Exception as e:
|
||||
log.error("Error encountered when trying to check latest MVT indicators: %s", e)
|
||||
else:
|
||||
if ioc_to_update:
|
||||
rich_print(
|
||||
"\t[bold]There are updates to your indicators files! "
|
||||
"Run the `download-iocs` command to update![/bold]"
|
||||
)
|
||||
else:
|
||||
rich_print("\tYour indicators files seem to be up to date.")
|
||||
|
||||
|
||||
def _print_plugin_updates(findings: list) -> None:
|
||||
if not findings:
|
||||
return
|
||||
|
||||
rich_print("\t[bold]Plugin updates available:[/bold]")
|
||||
for finding in findings:
|
||||
rich_print(
|
||||
f"\t {finding['name']} {finding['installed']} → "
|
||||
f"{finding['latest']} ({finding['upgrade_command']})"
|
||||
)
|
||||
|
||||
|
||||
def _check_plugin_updates(log: logging.Logger) -> None:
|
||||
if not settings.NETWORK_ACCESS_ALLOWED:
|
||||
return
|
||||
|
||||
# This runs on every command, so nothing here, including reading back what
|
||||
# the latest check stored, may ever interrupt MVT.
|
||||
try:
|
||||
distributions = installed_plugin_distributions()
|
||||
|
||||
# There is nothing to check when MVT was not extended with any plugin.
|
||||
if not distributions:
|
||||
return
|
||||
|
||||
plugin_updates = PluginUpdates()
|
||||
|
||||
# We only perform this check at a fixed frequency, in order to not
|
||||
# overburden the user (and the plugin repositories) with too many
|
||||
# lookups. In between checks we print the findings of the latest one,
|
||||
# leaving out those which no longer apply to what is installed.
|
||||
should_check, _ = plugin_updates.should_check()
|
||||
if not should_check:
|
||||
_print_plugin_updates(plugin_updates.current_findings(distributions))
|
||||
return
|
||||
|
||||
findings = plugin_updates.check()
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
rich_print(
|
||||
"\t[bold]Note: Could not check for plugin updates.[/bold] "
|
||||
"You may be working offline. Please update your MVT plugins regularly."
|
||||
)
|
||||
except Exception as e:
|
||||
log.error("Error encountered when trying to check MVT plugin updates: %s", e)
|
||||
else:
|
||||
_print_plugin_updates(findings)
|
||||
|
||||
|
||||
def check_updates(
|
||||
disable_version_check: bool = False, disable_indicator_check: bool = False
|
||||
) -> None:
|
||||
@@ -19,77 +134,25 @@ def check_updates(
|
||||
|
||||
# First we check for MVT version updates.
|
||||
if not disable_version_check:
|
||||
try:
|
||||
mvt_updates = MVTUpdates()
|
||||
latest_version = mvt_updates.check()
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
rich_print(
|
||||
"\t\t[bold]Note: Could not check for MVT updates.[/bold] "
|
||||
"You may be working offline. Please update MVT regularly."
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(
|
||||
"Error encountered when trying to check latest MVT version: %s", e
|
||||
)
|
||||
else:
|
||||
if latest_version:
|
||||
rich_print(
|
||||
f"\t\t[bold]Version {latest_version} is available! "
|
||||
"Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
|
||||
)
|
||||
_check_version_updates(log)
|
||||
|
||||
# Then we check for indicators files updates.
|
||||
if not disable_indicator_check:
|
||||
ioc_updates = IndicatorsUpdates()
|
||||
_check_indicator_updates(log)
|
||||
|
||||
# Before proceeding, we check if we have downloaded an indicators index.
|
||||
# If not, there's no point in proceeding with the updates check.
|
||||
if ioc_updates.get_latest_update() == 0:
|
||||
rich_print(
|
||||
"\t\t[bold]You have not yet downloaded any indicators, check "
|
||||
"the `download-iocs` command![/bold]"
|
||||
)
|
||||
return
|
||||
|
||||
# We only perform this check at a fixed frequency, in order to not
|
||||
# overburden the user with too many lookups if the command is being run
|
||||
# multiple times.
|
||||
should_check, hours = ioc_updates.should_check()
|
||||
if not should_check:
|
||||
rich_print(
|
||||
f"\t\tIndicators updates checked recently, next automatic check "
|
||||
f"in {int(hours)} hours"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
ioc_to_update = ioc_updates.check()
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
rich_print(
|
||||
"\t\t[bold]Note: Could not check for indicator updates.[/bold] "
|
||||
"You may be working offline. Please update MVT indicators regularly."
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(
|
||||
"Error encountered when trying to check latest MVT indicators: %s", e
|
||||
)
|
||||
else:
|
||||
if ioc_to_update:
|
||||
rich_print(
|
||||
"\t\t[bold]There are updates to your indicators files! "
|
||||
"Run the `download-iocs` command to update![/bold]"
|
||||
)
|
||||
else:
|
||||
rich_print("\t\tYour indicators files seem to be up to date.")
|
||||
# Finally we check for updates to the installed plugin packages. MVT never
|
||||
# installs an update itself, it only reports the command which does.
|
||||
if not disable_version_check:
|
||||
_check_plugin_updates(log)
|
||||
|
||||
|
||||
def logo(
|
||||
disable_version_check: bool = False, disable_indicator_check: bool = False
|
||||
) -> None:
|
||||
rich_print("\n")
|
||||
rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
|
||||
rich_print("\t\thttps://mvt.re")
|
||||
rich_print(f"\t\tVersion: {MVT_VERSION}")
|
||||
rich_print("\t[bold]MVT - Mobile Verification Toolkit[/bold]\n")
|
||||
rich_print("\thttps://mvt.re")
|
||||
rich_print(f"\tVersion: {MVT_VERSION}\n")
|
||||
|
||||
check_updates(disable_version_check, disable_indicator_check)
|
||||
|
||||
|
||||
@@ -4,18 +4,28 @@
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from .module import MVTModule
|
||||
from .version import MVT_VERSION
|
||||
|
||||
MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES"
|
||||
MODULES_ENTRY_POINT_GROUP = "mvt.modules"
|
||||
EXTERNAL_LOGGER_NAMESPACE = "mvt.ext"
|
||||
PLUGIN_PACKAGE_PREFIX = "mvt_plugin_"
|
||||
_ORIGIN_ATTRIBUTE = "_mvt_module_origin"
|
||||
_PATH_MODULE_PREFIX = "_mvt_custom_module_"
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -23,15 +33,74 @@ class CustomModuleLoadError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModuleOrigin:
|
||||
"""Describes where a loaded module came from, for auditability.
|
||||
|
||||
``kind`` is one of ``builtin`` (shipped with MVT), ``package`` (loaded
|
||||
from an installed package) or ``path`` (loaded from a file passed with
|
||||
``--load-module`` or the environment variable).
|
||||
"""
|
||||
|
||||
kind: str
|
||||
name: str
|
||||
version: Optional[str] = None
|
||||
commit: Optional[str] = None
|
||||
file_sha256: Optional[str] = None
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
label = self.name
|
||||
if self.version:
|
||||
label += f"@{self.version}"
|
||||
label = f"'{label}'"
|
||||
if self.commit:
|
||||
label += f" (commit {self.commit})"
|
||||
if self.file_sha256:
|
||||
label += f" (sha256: {self.file_sha256})"
|
||||
return label
|
||||
|
||||
|
||||
def _module_name_for_path(path: Path) -> str:
|
||||
digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16]
|
||||
return f"_mvt_custom_module_{path.stem}_{digest}"
|
||||
return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}"
|
||||
|
||||
|
||||
def get_module_logger(module_class: type[MVTModule]) -> logging.Logger:
|
||||
"""Return the logger a module's records should be emitted through.
|
||||
|
||||
Modules loaded from installed packages or file paths live outside the
|
||||
"mvt" logger hierarchy, so their records would never reach the handlers
|
||||
attached to the "mvt" logger and instead fall through to
|
||||
logging.lastResort (which prints bare messages and drops anything below
|
||||
WARNING). Their loggers are parented under the "mvt.ext" namespace,
|
||||
keeping external module names from colliding with MVT's own logger
|
||||
tree. File-path modules are named after their file instead of the
|
||||
mangled internal import name, and packages following the recommended
|
||||
"mvt_plugin_<name>" naming convention log under "mvt.ext.<name>".
|
||||
"""
|
||||
name = module_class.__module__
|
||||
if name == "mvt" or name.startswith("mvt."):
|
||||
return logging.getLogger(name)
|
||||
|
||||
if name.startswith(_PATH_MODULE_PREFIX):
|
||||
name = Path(get_module_origin(module_class).name).stem
|
||||
else:
|
||||
top_level, separator, rest = name.partition(".")
|
||||
if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len(
|
||||
PLUGIN_PACKAGE_PREFIX
|
||||
):
|
||||
name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest
|
||||
|
||||
return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{name}")
|
||||
|
||||
|
||||
def _iter_module_files(path: Path) -> Iterable[Path]:
|
||||
if path.is_file():
|
||||
if path.suffix != ".py":
|
||||
raise CustomModuleLoadError(f"Custom module file is not a Python file: {path}")
|
||||
raise CustomModuleLoadError(
|
||||
f"Custom module file is not a Python file: {path}"
|
||||
)
|
||||
yield path
|
||||
return
|
||||
|
||||
@@ -59,7 +128,9 @@ def _load_python_file(path: Path) -> ModuleType:
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception as exc:
|
||||
raise CustomModuleLoadError(f"Unable to import custom module {path}: {exc}") from exc
|
||||
raise CustomModuleLoadError(
|
||||
f"Unable to import custom module {path}: {exc}"
|
||||
) from exc
|
||||
|
||||
return module
|
||||
|
||||
@@ -84,17 +155,171 @@ def load_custom_modules_from_path(path: str) -> list[type[MVTModule]]:
|
||||
resolved_path = Path(path).expanduser().resolve()
|
||||
|
||||
for module_file in _iter_module_files(resolved_path):
|
||||
file_sha256 = hashlib.sha256(module_file.read_bytes()).hexdigest()
|
||||
loaded_module = _load_python_file(module_file)
|
||||
origin = ModuleOrigin(
|
||||
kind="path", name=str(module_file), file_sha256=file_sha256
|
||||
)
|
||||
for module_class in discover_mvt_modules(loaded_module):
|
||||
key = (str(module_file), module_class.__qualname__)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
setattr(module_class, _ORIGIN_ATTRIBUTE, origin)
|
||||
custom_modules.append(module_class)
|
||||
|
||||
return custom_modules
|
||||
|
||||
|
||||
def _module_key(module_class: type[MVTModule]) -> tuple[str, str]:
|
||||
try:
|
||||
source = str(Path(inspect.getfile(module_class)).resolve())
|
||||
except (OSError, TypeError):
|
||||
source = module_class.__module__
|
||||
return (source, module_class.__qualname__)
|
||||
|
||||
|
||||
def distribution_direct_url(dist: importlib.metadata.Distribution) -> Optional[dict]:
|
||||
"""Return the PEP 610 direct URL metadata of a distribution, if recorded.
|
||||
|
||||
Packages installed from an index have no direct URL metadata, while
|
||||
packages installed directly from a repository or from a local folder
|
||||
record where they were installed from in ``direct_url.json``.
|
||||
"""
|
||||
try:
|
||||
direct_url_text = dist.read_text("direct_url.json")
|
||||
if not direct_url_text:
|
||||
return None
|
||||
direct_url = json.loads(direct_url_text)
|
||||
return direct_url if isinstance(direct_url, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]:
|
||||
"""Return the VCS commit a distribution was installed from, if recorded.
|
||||
|
||||
Packages installed directly from a repository (``pip install git+...``)
|
||||
record the commit in ``direct_url.json`` (PEP 610).
|
||||
"""
|
||||
vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info")
|
||||
if not isinstance(vcs_info, dict):
|
||||
return None
|
||||
|
||||
commit = vcs_info.get("commit_id")
|
||||
return commit if isinstance(commit, str) else None
|
||||
|
||||
|
||||
def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin:
|
||||
name = entry_point.name
|
||||
version = None
|
||||
commit = None
|
||||
# Manually constructed entry points have no associated distribution.
|
||||
dist = getattr(entry_point, "dist", None)
|
||||
if dist is not None:
|
||||
try:
|
||||
name = dist.name or name
|
||||
version = dist.version
|
||||
except Exception:
|
||||
pass
|
||||
commit = _distribution_commit(dist)
|
||||
return ModuleOrigin(kind="package", name=name, version=version, commit=commit)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _packages_distributions() -> dict[str, list[str]]:
|
||||
try:
|
||||
return dict(importlib.metadata.packages_distributions())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def get_module_origin(module_class: type[MVTModule]) -> ModuleOrigin:
|
||||
"""Return the origin of a module class for auditing purposes."""
|
||||
origin = module_class.__dict__.get(_ORIGIN_ATTRIBUTE)
|
||||
if isinstance(origin, ModuleOrigin):
|
||||
return origin
|
||||
|
||||
top_level = module_class.__module__.partition(".")[0]
|
||||
if top_level == "mvt":
|
||||
return ModuleOrigin(kind="builtin", name="mvt", version=MVT_VERSION)
|
||||
|
||||
distributions = _packages_distributions().get(top_level)
|
||||
if distributions:
|
||||
name = distributions[0]
|
||||
version = None
|
||||
commit = None
|
||||
try:
|
||||
dist = importlib.metadata.distribution(name)
|
||||
version = dist.version
|
||||
commit = _distribution_commit(dist)
|
||||
except Exception:
|
||||
pass
|
||||
return ModuleOrigin(kind="package", name=name, version=version, commit=commit)
|
||||
|
||||
try:
|
||||
source = str(Path(inspect.getfile(module_class)).resolve())
|
||||
except (OSError, TypeError):
|
||||
source = module_class.__module__
|
||||
return ModuleOrigin(kind="path", name=source)
|
||||
|
||||
|
||||
def load_installed_modules() -> list[type[MVTModule]]:
|
||||
"""Load MVT modules registered by installed packages.
|
||||
|
||||
Packages register modules in the ``mvt.modules`` entry-point group. Each
|
||||
entry point must resolve to an iterable of MVTModule subclasses, or to a
|
||||
callable which returns one. A broken entry point is skipped with a
|
||||
warning so that a faulty plugin package cannot break MVT.
|
||||
"""
|
||||
try:
|
||||
entry_points = importlib.metadata.entry_points(group=MODULES_ENTRY_POINT_GROUP)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Unable to discover installed module packages in entry-point group %s: %s",
|
||||
MODULES_ENTRY_POINT_GROUP,
|
||||
exc,
|
||||
)
|
||||
return []
|
||||
|
||||
installed_modules: list[type[MVTModule]] = []
|
||||
ordered_entry_points = sorted(
|
||||
entry_points, key=lambda entry_point: (entry_point.name, entry_point.value)
|
||||
)
|
||||
for entry_point in ordered_entry_points:
|
||||
try:
|
||||
loaded = entry_point.load()
|
||||
if callable(loaded) and not isinstance(loaded, type):
|
||||
loaded = loaded()
|
||||
module_classes = list(loaded)
|
||||
except (Exception, SystemExit) as exc:
|
||||
log.warning(
|
||||
"Unable to load modules from entry point %s (%s): %s",
|
||||
entry_point.name,
|
||||
entry_point.value,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
origin = _entry_point_origin(entry_point)
|
||||
for module_class in module_classes:
|
||||
if not (
|
||||
isinstance(module_class, type) and issubclass(module_class, MVTModule)
|
||||
):
|
||||
log.warning(
|
||||
"Entry point %s (%s) provided %r which is not an "
|
||||
"MVTModule subclass",
|
||||
entry_point.name,
|
||||
entry_point.value,
|
||||
module_class,
|
||||
)
|
||||
continue
|
||||
setattr(module_class, _ORIGIN_ATTRIBUTE, origin)
|
||||
installed_modules.append(module_class)
|
||||
|
||||
return installed_modules
|
||||
|
||||
|
||||
def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTModule]]:
|
||||
search_paths: list[str] = []
|
||||
env_path = os.environ.get(MVT_CUSTOM_MODULES_ENV)
|
||||
@@ -105,10 +330,17 @@ def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTM
|
||||
|
||||
custom_modules: list[type[MVTModule]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
for module_class in load_installed_modules():
|
||||
key = _module_key(module_class)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
custom_modules.append(module_class)
|
||||
|
||||
for path in search_paths:
|
||||
for module_class in load_custom_modules_from_path(path):
|
||||
source = Path(inspect.getfile(module_class)).resolve()
|
||||
key = (str(source), module_class.__qualname__)
|
||||
key = _module_key(module_class)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Any, ClassVar, Dict, List, Tuple, Type, TypeVar
|
||||
|
||||
import yaml
|
||||
from appdirs import user_config_dir, user_data_dir
|
||||
from pydantic import ValidationError
|
||||
from pydantic_settings import (
|
||||
BaseSettings,
|
||||
EnvSettingsSource,
|
||||
PydanticBaseSettingsSource,
|
||||
SettingsConfigDict,
|
||||
YamlConfigSettingsSource,
|
||||
)
|
||||
|
||||
PLUGIN_CONFIG_FOLDER_NAME = "plugins"
|
||||
# Not "plugins": on macOS the configuration and data folders are the same
|
||||
# directory, and that name already holds the settings files.
|
||||
PLUGIN_DATA_FOLDER_NAME = "plugin-data"
|
||||
PLUGIN_ENV_PREFIX = "MVT_PLUGIN_"
|
||||
|
||||
PLUGIN_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]*")
|
||||
|
||||
PluginSettingsType = TypeVar("PluginSettingsType", bound="MVTPluginSettings")
|
||||
|
||||
|
||||
class PluginConfigLoadError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def validate_plugin_name(plugin_name: Any) -> str:
|
||||
"""
|
||||
Check that a plugin name is safe to use in a file name and an env variable.
|
||||
|
||||
:param plugin_name: Name to validate.
|
||||
:returns: The validated plugin name.
|
||||
"""
|
||||
if not isinstance(plugin_name, str):
|
||||
raise TypeError(
|
||||
f"Plugin name must be a string, not {type(plugin_name).__name__}"
|
||||
)
|
||||
if not PLUGIN_NAME_PATTERN.fullmatch(plugin_name):
|
||||
raise ValueError(
|
||||
f"Invalid plugin name {plugin_name!r}: plugin names must start with a "
|
||||
"lowercase letter or a digit and may only contain lowercase letters, "
|
||||
"digits and dashes"
|
||||
)
|
||||
return plugin_name
|
||||
|
||||
|
||||
def plugin_config_folder() -> str:
|
||||
"""
|
||||
Return the folder where plugins store their configuration files.
|
||||
|
||||
The path is resolved on every call so it always reflects the current
|
||||
environment.
|
||||
"""
|
||||
return os.path.join(user_config_dir("mvt"), PLUGIN_CONFIG_FOLDER_NAME)
|
||||
|
||||
|
||||
def plugin_config_path(plugin_name: str) -> str:
|
||||
"""
|
||||
Return the path of the configuration file of a given plugin.
|
||||
|
||||
:param plugin_name: Name of the plugin.
|
||||
"""
|
||||
return os.path.join(
|
||||
plugin_config_folder(), f"{validate_plugin_name(plugin_name)}.yaml"
|
||||
)
|
||||
|
||||
|
||||
def plugin_data_folder(plugin_name: str) -> str:
|
||||
"""
|
||||
Return the folder where a given plugin stores its data, creating it.
|
||||
|
||||
Plugins should keep whatever they persist, such as caches or downloaded
|
||||
artifacts, in this folder. It is created with owner-only permissions. The
|
||||
path is resolved on every call so it always reflects the current
|
||||
environment. A plugin with a settings class calls
|
||||
`MVTPluginSettings.data_folder()` instead, which passes `plugin_name` here.
|
||||
|
||||
:param plugin_name: Name of the plugin.
|
||||
:returns: The path of the data folder of the plugin.
|
||||
"""
|
||||
# Validate the name before anything is created, so an unsafe name cannot
|
||||
# leave a folder behind.
|
||||
name = validate_plugin_name(plugin_name)
|
||||
|
||||
# makedirs() applies its mode only to the last folder of the path, so
|
||||
# MVT's own data folder keeps the default permissions while the two
|
||||
# plugin folders are private.
|
||||
data_folder = os.path.join(user_data_dir("mvt"), PLUGIN_DATA_FOLDER_NAME)
|
||||
os.makedirs(data_folder, mode=0o700, exist_ok=True)
|
||||
|
||||
folder = os.path.join(data_folder, name)
|
||||
os.makedirs(folder, mode=0o700, exist_ok=True)
|
||||
return folder
|
||||
|
||||
|
||||
def plugin_env_prefix(plugin_name: str) -> str:
|
||||
"""
|
||||
Return the environment variable prefix used by a given plugin.
|
||||
|
||||
Dashes are replaced by underscores. Plugin names cannot contain underscores,
|
||||
so two different plugin names never share an environment namespace.
|
||||
|
||||
:param plugin_name: Name of the plugin.
|
||||
"""
|
||||
name = validate_plugin_name(plugin_name).upper().replace("-", "_")
|
||||
return f"{PLUGIN_ENV_PREFIX}{name}_"
|
||||
|
||||
|
||||
def _settings_plugin_name(settings_cls: Type[BaseSettings]) -> str:
|
||||
plugin_name = getattr(settings_cls, "plugin_name", None)
|
||||
if plugin_name is None:
|
||||
raise TypeError(
|
||||
f"{settings_cls.__name__} must set a 'plugin_name' class attribute to "
|
||||
"namespace its configuration file and environment variables"
|
||||
)
|
||||
return validate_plugin_name(plugin_name)
|
||||
|
||||
|
||||
def _plugin_yaml_source(
|
||||
settings_cls: Type[BaseSettings], config_path: str
|
||||
) -> YamlConfigSettingsSource:
|
||||
"""
|
||||
Build the YAML settings source of a plugin, reporting unusable files.
|
||||
|
||||
A missing file is not an error, but a file which cannot be parsed or which
|
||||
does not hold a mapping of setting names is reported with its path.
|
||||
"""
|
||||
try:
|
||||
return YamlConfigSettingsSource(settings_cls, config_path)
|
||||
except yaml.YAMLError as exc:
|
||||
raise PluginConfigLoadError(
|
||||
f"Invalid plugin configuration file {config_path}: {exc}"
|
||||
) from exc
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise PluginConfigLoadError(
|
||||
f"Invalid plugin configuration file {config_path}: the file must "
|
||||
"contain a mapping of setting names to values"
|
||||
) from exc
|
||||
|
||||
|
||||
class MVTPluginSettings(BaseSettings):
|
||||
"""
|
||||
Base class for plugin-namespaced settings.
|
||||
|
||||
Subclass with typed fields and set `plugin_name`. Values resolve from
|
||||
constructor arguments, then environment variables (MVT_PLUGIN_<NAME>_*),
|
||||
then the plugin's YAML file (~/.config/mvt/plugins/<name>.yaml), then field
|
||||
defaults.
|
||||
|
||||
Plugins must not store their settings in MVT's own configuration file: MVT
|
||||
rewrites it with the fields it knows about, dropping anything else.
|
||||
|
||||
`data_folder()` returns the folder the plugin keeps its data in.
|
||||
"""
|
||||
|
||||
model_config = SettingsConfigDict(extra="ignore")
|
||||
|
||||
plugin_name: ClassVar[str]
|
||||
|
||||
def __init_subclass__(cls, **kwargs: Any) -> None:
|
||||
super().__init_subclass__(**kwargs)
|
||||
plugin_name = _settings_plugin_name(cls)
|
||||
# Namespace the environment variables of this plugin. Each pydantic
|
||||
# model gets its own configuration dictionary, so this does not leak
|
||||
# into other plugins.
|
||||
cls.model_config["env_prefix"] = plugin_env_prefix(plugin_name)
|
||||
|
||||
@classmethod
|
||||
def settings_customise_sources(
|
||||
cls,
|
||||
settings_cls: Type[BaseSettings],
|
||||
init_settings: PydanticBaseSettingsSource,
|
||||
env_settings: PydanticBaseSettingsSource,
|
||||
dotenv_settings: PydanticBaseSettingsSource,
|
||||
file_secret_settings: PydanticBaseSettingsSource,
|
||||
) -> Tuple[PydanticBaseSettingsSource, ...]:
|
||||
config_path = plugin_config_path(_settings_plugin_name(settings_cls))
|
||||
yaml_source = _plugin_yaml_source(settings_cls, config_path)
|
||||
# Explicit arguments take precedence over environment variables, which
|
||||
# in turn take precedence over the configuration file.
|
||||
return (init_settings, env_settings, yaml_source)
|
||||
|
||||
@classmethod
|
||||
def load(cls: Type[PluginSettingsType]) -> PluginSettingsType:
|
||||
"""
|
||||
Load the settings of the plugin.
|
||||
|
||||
A missing configuration file is not an error: the settings then come
|
||||
from the environment and from the field defaults.
|
||||
"""
|
||||
return cls()
|
||||
|
||||
@classmethod
|
||||
def data_folder(cls) -> str:
|
||||
"""
|
||||
Return the data folder of the plugin, creating it.
|
||||
|
||||
The folder is the one plugin_data_folder() returns for `plugin_name`,
|
||||
so a plugin with a settings class does not repeat its name.
|
||||
"""
|
||||
return plugin_data_folder(_settings_plugin_name(cls))
|
||||
|
||||
def _environment_values(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Return the settings values currently supplied by the environment.
|
||||
|
||||
Values are validated by the model, so they can be compared with the
|
||||
values held by this instance.
|
||||
"""
|
||||
settings_cls = type(self)
|
||||
raw_values = EnvSettingsSource(settings_cls)()
|
||||
names = [name for name in raw_values if name in settings_cls.model_fields]
|
||||
if not names:
|
||||
return {}
|
||||
|
||||
# Fall back on the current values for the fields the environment does
|
||||
# not set, so that required fields do not fail validation here.
|
||||
current_values = json.loads(self.model_dump_json())
|
||||
try:
|
||||
from_environment = settings_cls.model_validate(
|
||||
{**current_values, **raw_values}
|
||||
)
|
||||
except ValidationError:
|
||||
# An environment variable which the model cannot validate must not
|
||||
# stop the other environment values from being recognised, or a
|
||||
# credential would be written to the configuration file.
|
||||
return self._environment_values_by_field(current_values, raw_values, names)
|
||||
return {name: getattr(from_environment, name) for name in names}
|
||||
|
||||
def _environment_values_by_field(
|
||||
self,
|
||||
current_values: Dict[str, Any],
|
||||
raw_values: Dict[str, Any],
|
||||
names: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate each environment value on its own, skipping unusable ones.
|
||||
|
||||
:param current_values: Serialized values held by this instance.
|
||||
:param raw_values: Values supplied by the environment.
|
||||
:param names: Names of the fields set by the environment.
|
||||
"""
|
||||
settings_cls = type(self)
|
||||
values = {}
|
||||
for name in names:
|
||||
try:
|
||||
from_environment = settings_cls.model_validate(
|
||||
{**current_values, name: raw_values[name]}
|
||||
)
|
||||
except ValidationError:
|
||||
continue
|
||||
values[name] = getattr(from_environment, name)
|
||||
return values
|
||||
|
||||
def save(self) -> None:
|
||||
"""
|
||||
Save the current settings to the configuration file of the plugin.
|
||||
|
||||
Only values which differ from the field defaults are persisted. Values
|
||||
which come from the environment are not written to disk, so credentials
|
||||
passed as environment variables stay out of the configuration file.
|
||||
MVT's own configuration file is never modified.
|
||||
"""
|
||||
config_folder = plugin_config_folder()
|
||||
if not os.path.isdir(config_folder):
|
||||
os.makedirs(config_folder, mode=0o700, exist_ok=True)
|
||||
|
||||
values = json.loads(self.model_dump_json(exclude_defaults=True))
|
||||
for name, environment_value in self._environment_values().items():
|
||||
if name in values and getattr(self, name, None) == environment_value:
|
||||
del values[name]
|
||||
|
||||
# Settings files can hold credentials, so write them through a private
|
||||
# temporary file and move it in place. The file is then never partially
|
||||
# written and never briefly readable by other users.
|
||||
config_path = plugin_config_path(self.plugin_name)
|
||||
descriptor, temporary_path = tempfile.mkstemp(
|
||||
dir=config_folder, prefix=f".{self.plugin_name}-", suffix=".yaml"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w") as config_file:
|
||||
config_file.write(yaml.dump(values, default_flow_style=False))
|
||||
os.replace(temporary_path, config_path)
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temporary_path)
|
||||
raise
|
||||
+416
-3
@@ -3,8 +3,13 @@
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import importlib.metadata
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple
|
||||
|
||||
@@ -12,14 +17,35 @@ import requests
|
||||
import yaml
|
||||
from packaging import version
|
||||
|
||||
from .cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
from .config import settings
|
||||
from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER
|
||||
from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url
|
||||
from .version import MVT_VERSION
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# In hours.
|
||||
INDICATORS_CHECK_FREQUENCY = 12
|
||||
PLUGINS_CHECK_FREQUENCY = 12
|
||||
|
||||
# The entry-point groups a package can use to extend MVT.
|
||||
PLUGIN_ENTRY_POINT_GROUPS = (
|
||||
MODULES_ENTRY_POINT_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
SHORT_COMMIT_LENGTH = 8
|
||||
# The keys every cached finding has to carry to be printed.
|
||||
FINDING_KEYS = ("name", "installed", "latest", "upgrade_command")
|
||||
# Options which stop ssh from waiting for an answer nobody is there to give.
|
||||
_SSH_BATCH_OPTIONS = ("-o", "BatchMode=yes", "-o", "ConnectTimeout=10")
|
||||
_COMMIT_PATTERN = re.compile(r"\A[0-9a-f]{7,40}\Z")
|
||||
|
||||
|
||||
class MVTUpdates:
|
||||
@@ -38,6 +64,395 @@ class MVTUpdates:
|
||||
return ""
|
||||
|
||||
|
||||
def installed_plugin_distributions() -> list[importlib.metadata.Distribution]:
|
||||
"""Return the installed distributions which extend MVT.
|
||||
|
||||
A plugin package is any distribution registering at least one entry point
|
||||
in the module or CLI command groups. Distributions are returned once each,
|
||||
sorted by name. MVT itself is not a plugin and is never returned.
|
||||
"""
|
||||
distributions: dict[str, importlib.metadata.Distribution] = {}
|
||||
|
||||
for group in PLUGIN_ENTRY_POINT_GROUPS:
|
||||
try:
|
||||
entry_points = importlib.metadata.entry_points(group=group)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Unable to discover installed plugin packages in entry-point "
|
||||
"group %s: %s",
|
||||
group,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
for entry_point in entry_points:
|
||||
# Manually constructed entry points have no associated distribution.
|
||||
dist = getattr(entry_point, "dist", None)
|
||||
if dist is None:
|
||||
continue
|
||||
try:
|
||||
name = dist.name
|
||||
except Exception:
|
||||
continue
|
||||
if not name or name == "mvt":
|
||||
continue
|
||||
distributions.setdefault(name, dist)
|
||||
|
||||
return [distributions[name] for name in sorted(distributions)]
|
||||
|
||||
|
||||
def _is_usable_finding(finding: object) -> bool:
|
||||
"""Check that a cached finding carries everything needed to print it."""
|
||||
if not isinstance(finding, dict):
|
||||
return False
|
||||
|
||||
return all(
|
||||
isinstance(finding.get(key), str) and finding.get(key) for key in FINDING_KEYS
|
||||
)
|
||||
|
||||
|
||||
def _installed_revision(
|
||||
dist: importlib.metadata.Distribution, origin: object
|
||||
) -> Optional[str]:
|
||||
"""Return what a plugin's installed revision is right now.
|
||||
|
||||
This is the value a finding recorded as installed when it was made, so a
|
||||
finding can be compared against the current state of the installation
|
||||
without looking anything up remotely.
|
||||
"""
|
||||
try:
|
||||
if origin != "git":
|
||||
return dist.version
|
||||
|
||||
vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info")
|
||||
if not isinstance(vcs_info, dict):
|
||||
return None
|
||||
|
||||
commit = vcs_info.get("commit_id") or ""
|
||||
return commit[:SHORT_COMMIT_LENGTH] or None
|
||||
except Exception as e:
|
||||
log.debug("Failed to read the installed revision of a plugin: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _batch_mode_ssh_command(ssh_command: str) -> str:
|
||||
"""Return an ssh command line which cannot stop to ask a question.
|
||||
|
||||
ssh keeps the first value it is given for a keyword, so the batch mode
|
||||
options are inserted right after the ssh program, ahead of whatever the
|
||||
analyst configured. Their remaining options, such as which key to use,
|
||||
still apply.
|
||||
"""
|
||||
try:
|
||||
arguments = shlex.split(ssh_command.strip())
|
||||
except ValueError:
|
||||
arguments = []
|
||||
|
||||
if not arguments:
|
||||
arguments = ["ssh"]
|
||||
|
||||
return shlex.join([arguments[0], *_SSH_BATCH_OPTIONS, *arguments[1:]])
|
||||
|
||||
|
||||
def _revision_pins_commit(revision: str, commit: str) -> bool:
|
||||
"""Check whether a requested revision pins the installed commit."""
|
||||
candidate = revision.lower()
|
||||
if not _COMMIT_PATTERN.match(candidate):
|
||||
return False
|
||||
|
||||
return commit.lower().startswith(candidate)
|
||||
|
||||
|
||||
class PluginUpdates:
|
||||
"""Check for updates to the installed MVT plugin packages.
|
||||
|
||||
MVT never installs or upgrades a plugin package itself. It only reports
|
||||
the command which upgrades a plugin, leaving the analyst to decide when to
|
||||
run it.
|
||||
"""
|
||||
|
||||
@property
|
||||
def latest_check_path(self) -> str:
|
||||
return os.path.join(MVT_DATA_FOLDER, "latest_plugins_check")
|
||||
|
||||
@property
|
||||
def findings_path(self) -> str:
|
||||
return os.path.join(MVT_DATA_FOLDER, "plugin_updates.json")
|
||||
|
||||
def _create_data_folder(self) -> None:
|
||||
if not os.path.exists(MVT_DATA_FOLDER):
|
||||
os.makedirs(MVT_DATA_FOLDER)
|
||||
|
||||
def get_latest_check(self) -> int:
|
||||
if not os.path.exists(self.latest_check_path):
|
||||
return 0
|
||||
|
||||
# A corrupt or truncated timestamp only means the next check happens
|
||||
# sooner. It must never stop MVT from running.
|
||||
try:
|
||||
with open(self.latest_check_path, "r", encoding="utf-8") as handle:
|
||||
data = handle.read().strip()
|
||||
if data:
|
||||
return int(data)
|
||||
except (OSError, ValueError) as e:
|
||||
log.debug("Failed to read the time of the latest plugin check: %s", e)
|
||||
|
||||
return 0
|
||||
|
||||
def set_latest_check(self) -> None:
|
||||
self._create_data_folder()
|
||||
timestamp = int(datetime.now().timestamp())
|
||||
with open(self.latest_check_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(str(timestamp))
|
||||
|
||||
def get_findings(self) -> list[dict]:
|
||||
"""
|
||||
Return the findings of the latest check, without checking again.
|
||||
Returns an empty list if no check was ever performed.
|
||||
"""
|
||||
if not os.path.exists(self.findings_path):
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(self.findings_path, "r", encoding="utf-8") as handle:
|
||||
findings = json.load(handle)
|
||||
except Exception as e:
|
||||
log.debug("Failed to read the cached plugin updates: %s", e)
|
||||
return []
|
||||
|
||||
if not isinstance(findings, list):
|
||||
return []
|
||||
|
||||
# Anything which does not look like a finding is dropped rather than
|
||||
# trusted: the cache is only a convenience.
|
||||
return [finding for finding in findings if _is_usable_finding(finding)]
|
||||
|
||||
def current_findings(
|
||||
self, distributions: Optional[list[importlib.metadata.Distribution]] = None
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Return the cached findings which still apply to what is installed.
|
||||
Findings about a plugin which was upgraded or removed since the latest
|
||||
check are dropped, so an update is never reported twice.
|
||||
"""
|
||||
if distributions is None:
|
||||
distributions = installed_plugin_distributions()
|
||||
|
||||
installed = {}
|
||||
for dist in distributions:
|
||||
try:
|
||||
installed[dist.name] = dist
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
current = []
|
||||
for finding in self.get_findings():
|
||||
plugin = installed.get(finding["name"])
|
||||
if plugin is None:
|
||||
continue
|
||||
if (
|
||||
_installed_revision(plugin, finding.get("origin"))
|
||||
!= finding["installed"]
|
||||
):
|
||||
continue
|
||||
current.append(finding)
|
||||
|
||||
return current
|
||||
|
||||
def set_findings(self, findings: list[dict]) -> None:
|
||||
self._create_data_folder()
|
||||
with open(self.findings_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(findings, handle)
|
||||
|
||||
def should_check(self) -> Tuple[bool, int]:
|
||||
"""
|
||||
Compare time of the latest plugins check with current time.
|
||||
Returns bool and number of hours since the last check.
|
||||
"""
|
||||
now = datetime.now()
|
||||
latest_check_ts = self.get_latest_check()
|
||||
latest_check_dt = datetime.fromtimestamp(latest_check_ts)
|
||||
|
||||
diff = now - latest_check_dt
|
||||
diff_hours = divmod(diff.total_seconds(), 3600)[0]
|
||||
|
||||
if diff_hours >= PLUGINS_CHECK_FREQUENCY:
|
||||
return True, 0
|
||||
|
||||
return False, int(PLUGINS_CHECK_FREQUENCY - diff_hours)
|
||||
|
||||
def _check_index_plugin(self, name: str, installed: str) -> Optional[dict]:
|
||||
"""Check a plugin installed from a package index for a newer release."""
|
||||
url = f"https://pypi.org/pypi/{name}/json"
|
||||
try:
|
||||
res = requests.get(url, timeout=settings.NETWORK_TIMEOUT)
|
||||
except requests.exceptions.RequestException as e:
|
||||
log.debug("Failed to check for updates to plugin %s: %s", name, e)
|
||||
return None
|
||||
|
||||
# Plugins which were never published to a public index are expected,
|
||||
# and there is nothing to compare their version against.
|
||||
if res.status_code == 404:
|
||||
return None
|
||||
|
||||
if res.status_code != 200:
|
||||
log.debug(
|
||||
"Failed to check for updates to plugin %s (error %d)",
|
||||
name,
|
||||
res.status_code,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
latest = res.json().get("info", {}).get("version", "")
|
||||
if not latest or version.parse(latest) <= version.parse(installed):
|
||||
return None
|
||||
except Exception as e:
|
||||
log.debug("Failed to compare the versions of plugin %s: %s", name, e)
|
||||
return None
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"installed": installed,
|
||||
"latest": latest,
|
||||
"origin": "pypi",
|
||||
# The name comes from package metadata, so the command MVT
|
||||
# suggests is quoted rather than assumed to be shell-safe.
|
||||
"upgrade_command": f"pip install -U {shlex.quote(name)}",
|
||||
}
|
||||
|
||||
def _git_ls_remote(self, url: str, revision: str) -> list[Tuple[str, str]]:
|
||||
"""Return the remote references matching a revision, if git allows it."""
|
||||
# Neither value is trusted: they are read from the metadata of an
|
||||
# installed package and must not turn into git options.
|
||||
if url.startswith("-") or revision.startswith("-"):
|
||||
log.debug("Skipping the update check for the invalid repository %s", url)
|
||||
return []
|
||||
|
||||
environment = dict(os.environ)
|
||||
# Never prompt the analyst for repository credentials. git handles its
|
||||
# own prompts, while ssh reads the terminal directly and only batch
|
||||
# mode makes it fail instead of asking.
|
||||
environment["GIT_TERMINAL_PROMPT"] = "0"
|
||||
environment["GIT_SSH_COMMAND"] = _batch_mode_ssh_command(
|
||||
environment.get("GIT_SSH_COMMAND", "")
|
||||
)
|
||||
|
||||
try:
|
||||
process = subprocess.run(
|
||||
["git", "ls-remote", url, revision],
|
||||
capture_output=True,
|
||||
stdin=subprocess.DEVNULL,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=settings.NETWORK_TIMEOUT,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
log.debug("Could not find git, skipping the update check for %s", url)
|
||||
return []
|
||||
except (subprocess.SubprocessError, OSError) as e:
|
||||
log.debug("Failed to query the repository %s: %s", url, e)
|
||||
return []
|
||||
|
||||
if process.returncode != 0:
|
||||
log.debug(
|
||||
"Failed to query the repository %s (error %d): %s",
|
||||
url,
|
||||
process.returncode,
|
||||
(process.stderr or "").strip(),
|
||||
)
|
||||
return []
|
||||
|
||||
references = []
|
||||
for line in (process.stdout or "").splitlines():
|
||||
commit, _, reference = line.partition("\t")
|
||||
if commit.strip() and reference.strip():
|
||||
references.append((commit.strip(), reference.strip()))
|
||||
|
||||
return references
|
||||
|
||||
def _check_repository_plugin(
|
||||
self, name: str, direct_url: dict, vcs_info: dict
|
||||
) -> Optional[dict]:
|
||||
"""Check a plugin installed from a repository for a newer commit."""
|
||||
url = direct_url.get("url") or ""
|
||||
installed = vcs_info.get("commit_id") or ""
|
||||
revision = vcs_info.get("requested_revision") or ""
|
||||
if not url or not installed:
|
||||
return None
|
||||
|
||||
# A plugin installed from a commit is pinned and never goes out of
|
||||
# date, no matter what the branch it came from does next.
|
||||
if revision and _revision_pins_commit(revision, installed):
|
||||
return None
|
||||
|
||||
latest = ""
|
||||
wanted_reference = f"refs/heads/{revision}" if revision else "HEAD"
|
||||
for commit, reference in self._git_ls_remote(url, revision or "HEAD"):
|
||||
# Tags are pinned installs too.
|
||||
if reference.startswith("refs/tags/"):
|
||||
return None
|
||||
if reference == wanted_reference:
|
||||
latest = commit
|
||||
|
||||
if not latest or latest == installed:
|
||||
return None
|
||||
|
||||
requirement = f"{name} @ git+{url}"
|
||||
if revision:
|
||||
requirement += f"@{revision}"
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"installed": installed[:SHORT_COMMIT_LENGTH],
|
||||
"latest": latest[:SHORT_COMMIT_LENGTH],
|
||||
"origin": "git",
|
||||
# A repository URL and a branch name can both hold characters a
|
||||
# shell would act on, so the requirement is quoted for the shell
|
||||
# the analyst is going to paste the command into.
|
||||
"upgrade_command": f"pip install -U {shlex.quote(requirement)}",
|
||||
}
|
||||
|
||||
def _check_distribution(
|
||||
self, dist: importlib.metadata.Distribution
|
||||
) -> Optional[dict]:
|
||||
try:
|
||||
name = dist.name
|
||||
installed = dist.version
|
||||
except Exception as e:
|
||||
log.debug("Failed to read the metadata of an installed plugin: %s", e)
|
||||
return None
|
||||
|
||||
direct_url = distribution_direct_url(dist)
|
||||
if direct_url is None:
|
||||
return self._check_index_plugin(name, installed)
|
||||
|
||||
vcs_info = direct_url.get("vcs_info")
|
||||
if isinstance(vcs_info, dict):
|
||||
return self._check_repository_plugin(name, direct_url, vcs_info)
|
||||
|
||||
# Plugins installed from a local folder, including editable installs,
|
||||
# are maintained by the analyst and have nothing to check against.
|
||||
return None
|
||||
|
||||
def check(self) -> list[dict]:
|
||||
"""
|
||||
Check every installed plugin package for an available update.
|
||||
Returns one entry per plugin which can be upgraded.
|
||||
"""
|
||||
findings = []
|
||||
for dist in installed_plugin_distributions():
|
||||
finding = self._check_distribution(dist)
|
||||
if finding:
|
||||
findings.append(finding)
|
||||
|
||||
self.set_findings(findings)
|
||||
self.set_latest_check()
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
class IndicatorsUpdates:
|
||||
def __init__(self) -> None:
|
||||
self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}"
|
||||
@@ -180,9 +595,7 @@ class IndicatorsUpdates:
|
||||
def _get_remote_file_latest_commit(
|
||||
self, owner: str, repo: str, branch: str, path: str
|
||||
) -> int:
|
||||
file_commit_url = (
|
||||
f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}"
|
||||
)
|
||||
file_commit_url = f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}"
|
||||
try:
|
||||
res = requests.get(file_commit_url, timeout=5)
|
||||
except requests.exceptions.RequestException as e:
|
||||
|
||||
@@ -239,6 +239,13 @@ def init_logging(verbose: bool = False):
|
||||
"""
|
||||
log = logging.getLogger("mvt")
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
# Importing an MVT CLI module calls init_logging() at import time, and
|
||||
# loaded module packages may import one indirectly. Keep this idempotent
|
||||
# so console log lines are not duplicated by a second handler.
|
||||
if any(isinstance(handler, MVTLogHandler) for handler in log.handlers):
|
||||
return
|
||||
|
||||
consoleHandler = MVTLogHandler()
|
||||
consoleHandler.setFormatter(logging.Formatter("%(message)s"))
|
||||
if verbose:
|
||||
|
||||
@@ -3,4 +3,4 @@
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
from .cli import cli
|
||||
from .cli import cli, main
|
||||
+30
-47
@@ -8,13 +8,13 @@ import logging
|
||||
import os
|
||||
|
||||
import click
|
||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||
from mvt.common.completion import (
|
||||
SUPPORTED_SHELLS,
|
||||
completion_instructions,
|
||||
generate_completion_script,
|
||||
install_completion_script,
|
||||
from mvt.common.cli_plugins import (
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||
load_cli_commands_option,
|
||||
register_cli_plugins,
|
||||
)
|
||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||
from mvt.common.logo import logo
|
||||
from mvt.common.options import MutuallyExclusiveOption
|
||||
from mvt.common.updates import IndicatorsUpdates
|
||||
@@ -45,7 +45,6 @@ from mvt.common.help import (
|
||||
HELP_MSG_CHECK_SYSDIAGNOSE,
|
||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||
HELP_MSG_COMPLETION,
|
||||
)
|
||||
from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
|
||||
from mvt.common.password import prompt_password
|
||||
@@ -86,6 +85,7 @@ def _load_custom_modules(load_module):
|
||||
# Main
|
||||
# ==============================================================================
|
||||
@click.group(invoke_without_command=False)
|
||||
@load_cli_commands_option
|
||||
@click.option(
|
||||
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
||||
)
|
||||
@@ -99,55 +99,20 @@ def cli(ctx, disable_update_check, disable_indicator_update_check):
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["disable_version_check"] = disable_update_check
|
||||
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
||||
if ctx.invoked_subcommand != "completion":
|
||||
logo(
|
||||
disable_version_check=disable_update_check,
|
||||
disable_indicator_check=disable_indicator_update_check,
|
||||
)
|
||||
logo(
|
||||
disable_version_check=disable_update_check,
|
||||
disable_indicator_check=disable_indicator_update_check,
|
||||
)
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: version
|
||||
# ==============================================================================
|
||||
@cli.command("version", help=HELP_MSG_VERSION)
|
||||
@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION)
|
||||
def version():
|
||||
return
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: completion
|
||||
# ==============================================================================
|
||||
@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION)
|
||||
@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS))
|
||||
@click.option(
|
||||
"--install",
|
||||
is_flag=True,
|
||||
help="Write completion files and update shell configuration.",
|
||||
)
|
||||
@click.pass_context
|
||||
def completion(ctx, shell, install):
|
||||
program_name = "mvt-ios"
|
||||
|
||||
if shell is None:
|
||||
if install:
|
||||
raise click.UsageError("A shell is required when using --install.")
|
||||
click.echo(completion_instructions(program_name))
|
||||
return
|
||||
|
||||
root_cli = ctx.find_root().command
|
||||
|
||||
if install:
|
||||
script_path = install_completion_script(root_cli, program_name, shell)
|
||||
click.echo(f"Installed {shell} completion to {script_path}")
|
||||
if shell in ("bash", "zsh"):
|
||||
click.echo(f"Updated ~/.{shell}rc")
|
||||
else:
|
||||
click.echo("Fish loads completion files automatically.")
|
||||
return
|
||||
|
||||
click.echo(generate_completion_script(root_cli, program_name, shell))
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Command: decrypt-backup
|
||||
# ==============================================================================
|
||||
@@ -523,3 +488,21 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
||||
def download_iocs():
|
||||
ioc_updates = IndicatorsUpdates()
|
||||
ioc_updates.update()
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Entry point of the mvt-ios console script
|
||||
# ==============================================================================
|
||||
def main() -> None:
|
||||
"""Register the external commands and run the mvt-ios CLI.
|
||||
|
||||
External commands are registered here rather than when this module is
|
||||
imported, so that importing MVT never runs third-party code and a plugin
|
||||
importing from MVT cannot re-enter a module that is still initializing.
|
||||
"""
|
||||
register_cli_plugins(
|
||||
cli,
|
||||
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
||||
environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
cli()
|
||||
@@ -6,8 +6,10 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any, Optional
|
||||
|
||||
from mvt.common.command import Command
|
||||
@@ -54,6 +56,8 @@ class CmdIOSCheckSysdiagnose(Command):
|
||||
self.sysdiagnose_archive: Optional[tarfile.TarFile] = None
|
||||
self.sysdiagnose_files: list[str] = []
|
||||
self.ips_files: list[dict[str, Any]] = []
|
||||
self.temp_sysdiagnose_dir: Optional[TemporaryDirectory[str]] = None
|
||||
self.extracted_sysdiagnose_path: Optional[str] = None
|
||||
|
||||
@staticmethod
|
||||
def _parse_bugtype_header(data: bytes) -> Optional[int]:
|
||||
@@ -91,21 +95,68 @@ class CmdIOSCheckSysdiagnose(Command):
|
||||
self.log.info("Parsing sysdiagnose archive. This might take a while...")
|
||||
self.sysdiagnose_format = "tar"
|
||||
self.sysdiagnose_archive = tarfile.open(self.target_path, "r:gz")
|
||||
for member in self.sysdiagnose_archive:
|
||||
self.sysdiagnose_files.append(member.name)
|
||||
if member.isfile() and member.name.endswith(".ips"):
|
||||
archive_handle = self.sysdiagnose_archive.extractfile(member)
|
||||
if archive_handle is not None:
|
||||
with archive_handle:
|
||||
self._add_ips_file(member.name, archive_handle.read())
|
||||
self._extract_sysdiagnose_archive()
|
||||
|
||||
def _extract_sysdiagnose_archive(self) -> None:
|
||||
archive = self.sysdiagnose_archive
|
||||
if archive is None:
|
||||
raise RuntimeError("Sysdiagnose archive has not been initialized")
|
||||
|
||||
self.temp_sysdiagnose_dir = TemporaryDirectory()
|
||||
extraction_root = Path(self.temp_sysdiagnose_dir.name).resolve()
|
||||
archive_roots = set()
|
||||
|
||||
for member in archive:
|
||||
member_path = PurePosixPath(member.name.replace("\\", "/"))
|
||||
if member_path.is_absolute() or ".." in member_path.parts:
|
||||
self.log.warning("Skipping unsafe sysdiagnose path %r", member.name)
|
||||
continue
|
||||
|
||||
destination = extraction_root.joinpath(*member_path.parts).resolve()
|
||||
if not destination.is_relative_to(extraction_root):
|
||||
self.log.warning("Skipping unsafe sysdiagnose path %r", member.name)
|
||||
continue
|
||||
|
||||
if not member_path.parts:
|
||||
continue
|
||||
archive_roots.add(member_path.parts[0])
|
||||
|
||||
if member.isdir():
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
|
||||
# Modules only need directories and regular files. Do not materialize
|
||||
# links or device nodes from an untrusted sysdiagnose archive.
|
||||
if not member.isfile():
|
||||
self.log.warning("Skipping unsafe sysdiagnose member %r", member.name)
|
||||
continue
|
||||
|
||||
normalized_name = member_path.as_posix()
|
||||
self.sysdiagnose_files.append(normalized_name)
|
||||
|
||||
source = archive.extractfile(member)
|
||||
if source is None:
|
||||
continue
|
||||
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with source, destination.open("wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
|
||||
if normalized_name.endswith(".ips"):
|
||||
self._add_ips_file(str(destination), destination.read_bytes())
|
||||
|
||||
if len(archive_roots) != 1:
|
||||
raise ValueError("Sysdiagnose archive must contain one top-level directory")
|
||||
|
||||
self.extracted_sysdiagnose_path = str(extraction_root / archive_roots.pop())
|
||||
|
||||
def module_init(self, module) -> None:
|
||||
module.ips_files = self.ips_files
|
||||
if self.sysdiagnose_format == "tar":
|
||||
if self.sysdiagnose_archive is None:
|
||||
raise RuntimeError("Sysdiagnose archive has not been initialized")
|
||||
module.from_sysdiagnose_tar(
|
||||
self.sysdiagnose_archive, self.sysdiagnose_files
|
||||
if self.extracted_sysdiagnose_path is None:
|
||||
raise RuntimeError("Sysdiagnose archive has not been extracted")
|
||||
module.from_sysdiagnose_folder(
|
||||
self.extracted_sysdiagnose_path, self.sysdiagnose_files
|
||||
)
|
||||
return
|
||||
if self.sysdiagnose_format == "dir" and self.target_path:
|
||||
@@ -117,3 +168,6 @@ class CmdIOSCheckSysdiagnose(Command):
|
||||
if self.sysdiagnose_archive is not None:
|
||||
self.sysdiagnose_archive.close()
|
||||
self.sysdiagnose_archive = None
|
||||
if self.temp_sysdiagnose_dir is not None:
|
||||
self.temp_sysdiagnose_dir.cleanup()
|
||||
self.temp_sysdiagnose_dir = None
|
||||
@@ -1216,6 +1216,14 @@
|
||||
"version": "18.7.9",
|
||||
"build": "22H355"
|
||||
},
|
||||
{
|
||||
"version": "18.7.10",
|
||||
"build": "22H373"
|
||||
},
|
||||
{
|
||||
"version": "18.7.10",
|
||||
"build": "22H374"
|
||||
},
|
||||
{
|
||||
"version": "26",
|
||||
"build": "23A341"
|
||||
@@ -1255,5 +1263,13 @@
|
||||
{
|
||||
"version": "26.5",
|
||||
"build": "23F77"
|
||||
},
|
||||
{
|
||||
"version": "26.6.1",
|
||||
"build": "23G82"
|
||||
},
|
||||
{
|
||||
"version": "26.6.1",
|
||||
"build": "23G83"
|
||||
}
|
||||
]
|
||||
@@ -26,6 +26,7 @@ from .tcc import TCC
|
||||
from .webkit_resource_load_statistics import WebkitResourceLoadStatistics
|
||||
from .webkit_session_resource_log import WebkitSessionResourceLog
|
||||
from .whatsapp import Whatsapp
|
||||
from .whatsapp_contacts import WhatsappContacts
|
||||
|
||||
MIXED_MODULES = [
|
||||
Calls,
|
||||
@@ -47,6 +48,7 @@ MIXED_MODULES = [
|
||||
WebkitResourceLoadStatistics,
|
||||
WebkitSessionResourceLog,
|
||||
Whatsapp,
|
||||
WhatsappContacts,
|
||||
Shortcuts,
|
||||
Applications,
|
||||
Calendar,
|
||||
|
||||
@@ -73,7 +73,9 @@ class Calls(IOSExtraction):
|
||||
"duration": row[1],
|
||||
"location": row[2],
|
||||
"number": (
|
||||
row[3].decode("utf-8") if row[3] and row[3] is bytes else row[3]
|
||||
row[3].decode("utf-8")
|
||||
if isinstance(row[3], bytes)
|
||||
else row[3]
|
||||
),
|
||||
"provider": row[4],
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from mvt.common.module_types import (
|
||||
ModuleAtomicResult,
|
||||
@@ -15,6 +17,7 @@ from mvt.common.module_types import (
|
||||
from mvt.common.utils import convert_mactime_to_iso
|
||||
|
||||
from ..base import IOSExtraction
|
||||
from .whatsapp_contacts import WhatsappContacts
|
||||
|
||||
INTERACTIONC_BACKUP_IDS = [
|
||||
"1f5a521220a3ad80ebfdc196978df8e7a2e49dee",
|
||||
@@ -22,6 +25,49 @@ INTERACTIONC_BACKUP_IDS = [
|
||||
INTERACTIONC_ROOT_PATHS = [
|
||||
"private/var/mobile/Library/CoreDuet/People/interactionC.db",
|
||||
]
|
||||
|
||||
# The interaction record's creation date normally trails its start date by
|
||||
# milliseconds: emitting it as a timeline event only duplicates the start
|
||||
# date event. A large divergence, however, indicates the record was
|
||||
# backfilled (sync, restore, or tampering) and is worth surfacing.
|
||||
CREATION_DATE_DIVERGENCE_THRESHOLD = 3600.0
|
||||
|
||||
# Per-contact aggregate dates from ZCONTACTS are repeated on every
|
||||
# interaction row of the same contact. They are serialized with a
|
||||
# contact-centric data string so that timeline de-duplication collapses
|
||||
# them into one event per contact.
|
||||
CONTACT_EVENT_TEMPLATES = {
|
||||
"contacts_creation_date": "Contact {party} first recorded in interactionC",
|
||||
"first_incoming_sender_date": "First incoming interaction from {party}",
|
||||
"last_incoming_sender_date": "Last incoming interaction from {party}",
|
||||
"first_incoming_recipient_date": (
|
||||
"First incoming interaction where {party} was a recipient"
|
||||
),
|
||||
"last_incoming_recipient_date": (
|
||||
"Last incoming interaction where {party} was a recipient"
|
||||
),
|
||||
"first_outgoing_recipient_date": (
|
||||
"First outgoing interaction to {party}"
|
||||
),
|
||||
"last_outgoing_recipient_date": (
|
||||
"Last outgoing interaction to {party}"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _parse_iso(timestamp) -> Optional[datetime.datetime]:
|
||||
try:
|
||||
return datetime.datetime.strptime(
|
||||
timestamp, "%Y-%m-%d %H:%M:%S.%f"
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _describe_delta(seconds: float) -> str:
|
||||
if seconds >= 86400:
|
||||
return f"{seconds / 86400:.0f} days"
|
||||
return f"{seconds / 3600:.0f} hours"
|
||||
# Taken from APOLLO
|
||||
# https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt
|
||||
QUERIES = [
|
||||
@@ -34,7 +80,7 @@ QUERIES = [
|
||||
CASE ZINTERACTIONS.ZDIRECTION
|
||||
WHEN '0' THEN 'INCOMING'
|
||||
WHEN '1' THEN 'OUTGOING'
|
||||
END 'DIRECTION' AS "direction",
|
||||
END AS "direction",
|
||||
ZCONTACTS.ZDISPLAYNAME AS "sender_display_name",
|
||||
ZCONTACTS.ZIDENTIFIER AS "sender_identifier",
|
||||
ZCONTACTS.ZPERSONID AS "sender_personid",
|
||||
@@ -89,7 +135,7 @@ QUERIES = [
|
||||
CASE ZINTERACTIONS.ZDIRECTION
|
||||
WHEN '0' THEN 'INCOMING'
|
||||
WHEN '1' THEN 'OUTGOING'
|
||||
END 'DIRECTION' AS "direction",
|
||||
END AS "direction",
|
||||
ZCONTACTS.ZDISPLAYNAME AS "sender_display_name",
|
||||
ZCONTACTS.ZIDENTIFIER AS "sender_identifier",
|
||||
ZCONTACTS.ZPERSONID AS "sender_personid",
|
||||
@@ -117,7 +163,7 @@ QUERIES = [
|
||||
CASE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE
|
||||
WHEN '0' THEN '0'
|
||||
ELSE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE
|
||||
END 'LAST INCOMING RECIPIENT DATE' AS "last_incoming_recipient_date",
|
||||
END AS "last_incoming_recipient_date",
|
||||
ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE AS "last_outgoing_recipient_date",
|
||||
ZCONTACTS.ZCUSTOMIDENTIFIER AS "custom_id",
|
||||
ZINTERACTIONS.ZCONTENTURL AS "interaction_content_url",
|
||||
@@ -218,9 +264,16 @@ QUERIES = [
|
||||
]
|
||||
|
||||
|
||||
WHATSAPP_BUNDLE_ID = "net.whatsapp.WhatsApp"
|
||||
|
||||
|
||||
class InteractionC(IOSExtraction):
|
||||
"""This module extracts data from InteractionC db."""
|
||||
|
||||
# WhatsApp identifies chat peers by LID in interactionC.db, which only the
|
||||
# WhatsApp contacts database can map back to a phone number and name.
|
||||
dependencies = [WhatsappContacts]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: Optional[str] = None,
|
||||
@@ -252,10 +305,51 @@ class InteractionC(IOSExtraction):
|
||||
"last_outgoing_recipient_date",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _describe_party(record: ModuleAtomicResult, prefix: str) -> Optional[str]:
|
||||
name = record.get(f"{prefix}_display_name") or record.get(
|
||||
f"{prefix}_resolved_name"
|
||||
)
|
||||
identifier = record.get(f"{prefix}_resolved_phone_number") or record.get(
|
||||
f"{prefix}_identifier"
|
||||
)
|
||||
if name and identifier:
|
||||
# A display name that is just a formatted copy of the phone
|
||||
# number adds no information.
|
||||
name_digits = re.sub(r"\D", "", name)
|
||||
if name_digits and name_digits == re.sub(r"\D", "", identifier):
|
||||
return identifier
|
||||
return f"{name} ({identifier})"
|
||||
return name or identifier or None
|
||||
|
||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||
sender = self._describe_party(record, "sender")
|
||||
# The chat peer from the domain identifier stands in when the
|
||||
# recipient was not recorded (or the recipient join is unavailable).
|
||||
recipient = self._describe_party(record, "recipient") or self._describe_party(
|
||||
record, "domain"
|
||||
)
|
||||
direction = record.get("direction")
|
||||
if not sender and direction == "OUTGOING":
|
||||
sender = "local user"
|
||||
if not recipient and direction == "INCOMING":
|
||||
recipient = "local user"
|
||||
|
||||
header = f"[{record['bundle_id']}]"
|
||||
if record.get("account"):
|
||||
header += f" {record['account']}"
|
||||
if direction:
|
||||
header += f" {direction}"
|
||||
|
||||
data = f"{header} from {sender or 'unknown'} to {recipient or 'unknown'}"
|
||||
if record.get("group_name"):
|
||||
data += f" (group: {record['group_name']})"
|
||||
if record.get("content"):
|
||||
data += f": {record['content']}"
|
||||
|
||||
records = []
|
||||
processed = []
|
||||
for timestamp in self.timestamps:
|
||||
for timestamp in ("start_date", "end_date"):
|
||||
# Check if the record has the current timestamp.
|
||||
if timestamp not in record or not record[timestamp]:
|
||||
continue
|
||||
@@ -269,16 +363,142 @@ class InteractionC(IOSExtraction):
|
||||
"timestamp": record[timestamp],
|
||||
"module": self.__class__.__name__,
|
||||
"event": timestamp,
|
||||
"data": f"[{record['bundle_id']}] {record['account']} - "
|
||||
f"from {record['sender_display_name']} ({record['sender_identifier']}) "
|
||||
f"to {record.get('recipient_display_name', '')} ({record.get('recipient_identifier', '')}):"
|
||||
f" {record.get('content', '')}",
|
||||
"data": data,
|
||||
}
|
||||
)
|
||||
processed.append(record[timestamp])
|
||||
|
||||
creation_event = self._serialize_creation_date(record, data)
|
||||
if creation_event:
|
||||
records.append(creation_event)
|
||||
|
||||
# Contact-level aggregates describe the sender's contact record.
|
||||
party = self._describe_party(record, "sender")
|
||||
if party:
|
||||
for field, template in CONTACT_EVENT_TEMPLATES.items():
|
||||
if not record.get(field):
|
||||
continue
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record[field],
|
||||
"module": self.__class__.__name__,
|
||||
"event": field,
|
||||
"data": template.format(party=party),
|
||||
}
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
def _serialize_creation_date(
|
||||
self, record: ModuleAtomicResult, data: str
|
||||
) -> Optional[dict]:
|
||||
"""Serialize the interaction record's creation date only when it
|
||||
diverges from the start date enough to indicate the record was
|
||||
backfilled."""
|
||||
creation = record.get("interactions_creation_date")
|
||||
if not creation:
|
||||
return None
|
||||
|
||||
event = {
|
||||
"timestamp": creation,
|
||||
"module": self.__class__.__name__,
|
||||
"event": "interactions_creation_date",
|
||||
"data": data,
|
||||
}
|
||||
|
||||
start = _parse_iso(record.get("start_date"))
|
||||
creation_parsed = _parse_iso(creation)
|
||||
if not start or not creation_parsed:
|
||||
# Without a start date the creation date is the only anchor.
|
||||
return event
|
||||
|
||||
delta = (creation_parsed - start).total_seconds()
|
||||
if abs(delta) < CREATION_DATE_DIVERGENCE_THRESHOLD:
|
||||
return None
|
||||
|
||||
direction = "after" if delta > 0 else "before"
|
||||
event["data"] = (
|
||||
f"Interaction record created {_describe_delta(abs(delta))} "
|
||||
f"{direction} the event: {data}"
|
||||
)
|
||||
return event
|
||||
|
||||
def _whatsapp_contact_maps(self) -> Tuple[dict, dict]:
|
||||
"""Build LID and phone-digit lookup maps from the WhatsappContacts
|
||||
module results, when available."""
|
||||
by_lid: dict = {}
|
||||
by_phone: dict = {}
|
||||
contacts_module = self.dependency_modules.get(WhatsappContacts)
|
||||
if not contacts_module:
|
||||
return by_lid, by_phone
|
||||
|
||||
for contact in contacts_module.results:
|
||||
name = contact.get("full_name") or contact.get("given_name")
|
||||
phone = contact.get("phone_number")
|
||||
entry = (phone, name)
|
||||
if contact.get("lid"):
|
||||
by_lid[contact["lid"]] = entry
|
||||
if phone:
|
||||
by_phone[re.sub(r"\D", "", phone)] = entry
|
||||
whatsapp_id = contact.get("whatsapp_id")
|
||||
if whatsapp_id and "@" in whatsapp_id:
|
||||
by_phone.setdefault(whatsapp_id.split("@")[0], entry)
|
||||
|
||||
return by_lid, by_phone
|
||||
|
||||
@staticmethod
|
||||
def _resolve_whatsapp_identifier(
|
||||
value, by_lid: dict, by_phone: dict
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Resolve a WhatsApp identifier (LID, JID or phone number) to a
|
||||
(phone_number, contact_name) tuple."""
|
||||
if not value:
|
||||
return None, None
|
||||
|
||||
value = str(value)
|
||||
if value.endswith("@lid"):
|
||||
return by_lid.get(value, (None, None))
|
||||
if value.endswith("@g.us"):
|
||||
return None, None
|
||||
if value.endswith("@s.whatsapp.net"):
|
||||
digits = value.split("@")[0]
|
||||
phone, name = by_phone.get(digits, (None, None))
|
||||
return phone or f"+{digits}", name
|
||||
if value.startswith("+"):
|
||||
_, name = by_phone.get(re.sub(r"\D", "", value), (None, None))
|
||||
return None, name
|
||||
|
||||
return None, None
|
||||
|
||||
def _postprocess_results(self) -> None:
|
||||
by_lid, by_phone = self._whatsapp_contact_maps()
|
||||
|
||||
for entry in self.results:
|
||||
# The fallback queries return ZDIRECTION raw instead of labelled.
|
||||
if entry.get("direction") in (0, "0"):
|
||||
entry["direction"] = "INCOMING"
|
||||
elif entry.get("direction") in (1, "1"):
|
||||
entry["direction"] = "OUTGOING"
|
||||
|
||||
if entry.get("bundle_id") != WHATSAPP_BUNDLE_ID:
|
||||
continue
|
||||
|
||||
candidates = {
|
||||
"sender": [entry.get("sender_identifier"), entry.get("custom_id")],
|
||||
"recipient": [entry.get("recipient_identifier")],
|
||||
"domain": [entry.get("domain_identifier")],
|
||||
}
|
||||
for prefix, values in candidates.items():
|
||||
phone = name = None
|
||||
for value in values:
|
||||
phone, name = self._resolve_whatsapp_identifier(
|
||||
value, by_lid, by_phone
|
||||
)
|
||||
if phone or name:
|
||||
break
|
||||
entry[f"{prefix}_resolved_phone_number"] = phone
|
||||
entry[f"{prefix}_resolved_name"] = name
|
||||
|
||||
def run(self) -> None:
|
||||
self._find_ios_database(
|
||||
backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS
|
||||
@@ -325,4 +545,6 @@ class InteractionC(IOSExtraction):
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
self._postprocess_results()
|
||||
|
||||
self.log.info("Extracted a total of %d InteractionC events", len(self.results))
|
||||
@@ -4,7 +4,9 @@
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Dict, Optional
|
||||
|
||||
from mvt.common.module_types import (
|
||||
ModuleAtomicResult,
|
||||
@@ -22,9 +24,71 @@ WHATSAPP_ROOT_PATHS = [
|
||||
"private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite",
|
||||
]
|
||||
|
||||
WHATSAPP_LID_BACKUP_IDS = [
|
||||
# SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-LID.sqlite"
|
||||
"e794f6ffcc3c222535f47684a63d5178da3c4500",
|
||||
]
|
||||
WHATSAPP_LID_ROOT_PATHS = [
|
||||
"private/var/mobile/Containers/Shared/AppGroup/*/LID.sqlite",
|
||||
]
|
||||
|
||||
# WhatsApp records the mapping between a contact's LID and phone number
|
||||
# identifiers in the ZWAPHONENUMBERLIDPAIR table. Depending on the WhatsApp
|
||||
# version this lives in a dedicated LID.sqlite database or in
|
||||
# ChatStorage.sqlite itself.
|
||||
LID_PAIRS_QUERY = """
|
||||
SELECT
|
||||
ZLID AS "lid",
|
||||
ZPHONENUMBER AS "phone_number",
|
||||
ZTIMESTAMP AS "pair_timestamp"
|
||||
FROM ZWAPHONENUMBERLIDPAIR;
|
||||
"""
|
||||
|
||||
CHAT_SESSIONS_QUERY = """
|
||||
SELECT
|
||||
ZWACHATSESSION.Z_PK AS "session_pk",
|
||||
ZWACHATSESSION.ZCONTACTJID AS "contact_jid",
|
||||
ZWACHATSESSION.ZPARTNERNAME AS "partner_name",
|
||||
ZWACHATSESSION.ZSESSIONTYPE AS "session_type",
|
||||
ZWACHATSESSION.ZARCHIVED AS "archived",
|
||||
ZWACHATSESSION.ZREMOVED AS "removed",
|
||||
ZWACHATSESSION.ZMESSAGECOUNTER AS "message_counter",
|
||||
ZWACHATSESSION.ZLASTMESSAGEDATE AS "last_message_date",
|
||||
ZWAGROUPINFO.ZCREATIONDATE AS "group_creation_date",
|
||||
MIN(ZWAMESSAGE.ZMESSAGEDATE) AS "first_stored_message_date",
|
||||
MAX(ZWAMESSAGE.ZMESSAGEDATE) AS "last_stored_message_date",
|
||||
COUNT(ZWAMESSAGE.Z_PK) AS "stored_message_count"
|
||||
FROM ZWACHATSESSION
|
||||
LEFT JOIN ZWAGROUPINFO
|
||||
ON ZWACHATSESSION.ZGROUPINFO = ZWAGROUPINFO.Z_PK
|
||||
LEFT JOIN ZWAMESSAGE
|
||||
ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK
|
||||
GROUP BY ZWACHATSESSION.Z_PK;
|
||||
"""
|
||||
|
||||
CHAT_SESSION_DATE_FIELDS = [
|
||||
"last_message_date",
|
||||
"group_creation_date",
|
||||
"first_stored_message_date",
|
||||
"last_stored_message_date",
|
||||
]
|
||||
|
||||
|
||||
def _describe_chat(record: ModuleAtomicResult) -> str:
|
||||
jid = record.get("contact_jid") or "unknown"
|
||||
name = record.get("partner_name")
|
||||
identifier = record.get("partner_resolved_phone_number") or jid
|
||||
label = f"'{name}' ({identifier})" if name else identifier
|
||||
is_group = jid.endswith("@g.us") or record.get("group_creation_date")
|
||||
if is_group:
|
||||
return f"WhatsApp group chat {label}"
|
||||
return f"WhatsApp chat with {label}"
|
||||
|
||||
|
||||
class Whatsapp(IOSExtraction):
|
||||
"""This module extracts all WhatsApp messages containing links."""
|
||||
"""This module extracts all WhatsApp messages containing links, as well
|
||||
as per-chat records with the first and last interaction dates of each
|
||||
conversation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -45,6 +109,21 @@ class Whatsapp(IOSExtraction):
|
||||
)
|
||||
|
||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||
if record.get("record_type") == "chat_session":
|
||||
return self._serialize_chat_session(record)
|
||||
if record.get("record_type") == "lid_phone_number_pair":
|
||||
if not record.get("pair_timestamp"):
|
||||
return []
|
||||
return {
|
||||
"timestamp": record["pair_timestamp"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "lid_pair_recorded",
|
||||
"data": (
|
||||
f"WhatsApp associated LID {record.get('lid')} with "
|
||||
f"phone number {record.get('phone_number')}"
|
||||
),
|
||||
}
|
||||
|
||||
text = record.get("ZTEXT", "").replace("\n", "\\n")
|
||||
links_text = ""
|
||||
if record.get("links"):
|
||||
@@ -57,6 +136,49 @@ class Whatsapp(IOSExtraction):
|
||||
"data": f"'{text}' from {record.get('ZFROMJID', 'Unknown')}{links_text}",
|
||||
}
|
||||
|
||||
def _serialize_chat_session(
|
||||
self, record: ModuleAtomicResult
|
||||
) -> ModuleSerializedResult:
|
||||
records = []
|
||||
chat = _describe_chat(record)
|
||||
|
||||
if record.get("group_creation_date"):
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record["group_creation_date"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "group_created",
|
||||
"data": f"{chat} was created",
|
||||
}
|
||||
)
|
||||
|
||||
if record.get("first_stored_message_date"):
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record["first_stored_message_date"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "chat_first_message",
|
||||
"data": f"First stored message in {chat}",
|
||||
}
|
||||
)
|
||||
|
||||
# The chat session's own last-message date is authoritative: it can
|
||||
# postdate the newest stored message if that message was deleted.
|
||||
last_message_date = record.get("last_message_date") or record.get(
|
||||
"last_stored_message_date"
|
||||
)
|
||||
if last_message_date:
|
||||
records.append(
|
||||
{
|
||||
"timestamp": last_message_date,
|
||||
"module": self.__class__.__name__,
|
||||
"event": "chat_last_message",
|
||||
"data": f"Last message in {chat}",
|
||||
}
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
def check_indicators(self) -> None:
|
||||
if not self.indicators:
|
||||
return
|
||||
@@ -146,7 +268,126 @@ class Whatsapp(IOSExtraction):
|
||||
message["links"] = list(set(filtered_links))
|
||||
self.results.append(message)
|
||||
|
||||
total_messages = len(self.results)
|
||||
lid_map = self._extract_lid_pairs(cur)
|
||||
total_sessions = self._extract_chat_sessions(cur, lid_map)
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
self.log.info("Extracted a total of %d WhatsApp messages", len(self.results))
|
||||
self.log.info(
|
||||
"Extracted a total of %d WhatsApp messages, %d chat sessions "
|
||||
"and %d LID-phone number pairs",
|
||||
total_messages,
|
||||
total_sessions,
|
||||
len(lid_map),
|
||||
)
|
||||
|
||||
def _find_lid_db_path(self) -> Optional[str]:
|
||||
for backup_id in WHATSAPP_LID_BACKUP_IDS:
|
||||
file_path = self._get_backup_file_from_id(backup_id)
|
||||
if file_path and os.path.exists(file_path):
|
||||
return file_path
|
||||
for found_path in self._get_fs_files_from_patterns(
|
||||
WHATSAPP_LID_ROOT_PATHS
|
||||
):
|
||||
return found_path
|
||||
return None
|
||||
|
||||
def _extract_lid_pairs(self, chat_cur: sqlite3.Cursor) -> Dict[str, str]:
|
||||
"""Extract the LID to phone number mapping from the dedicated
|
||||
LID.sqlite database, falling back to the same table in
|
||||
ChatStorage.sqlite. Returns a map of LID digits to phone number
|
||||
digits."""
|
||||
rows = []
|
||||
lid_db_path = self._find_lid_db_path()
|
||||
if lid_db_path:
|
||||
self.log.info(
|
||||
"Found WhatsApp LID database at path: %s", lid_db_path
|
||||
)
|
||||
lid_conn = self._open_sqlite_db(lid_db_path)
|
||||
try:
|
||||
lid_cur = lid_conn.cursor()
|
||||
lid_cur.execute(LID_PAIRS_QUERY)
|
||||
rows = lid_cur.fetchall()
|
||||
lid_cur.close()
|
||||
except sqlite3.DatabaseError as exc:
|
||||
self.log.warning(
|
||||
"Unable to extract WhatsApp LID-phone number pairs: %s",
|
||||
exc,
|
||||
)
|
||||
finally:
|
||||
lid_conn.close()
|
||||
else:
|
||||
try:
|
||||
chat_cur.execute(LID_PAIRS_QUERY)
|
||||
rows = chat_cur.fetchall()
|
||||
except sqlite3.OperationalError:
|
||||
self.log.info(
|
||||
"No WhatsApp LID database found in this backup or "
|
||||
"filesystem dump: LID chat identifiers cannot be "
|
||||
"resolved to phone numbers"
|
||||
)
|
||||
|
||||
lid_map: Dict[str, str] = {}
|
||||
for lid, phone_number, pair_timestamp in rows:
|
||||
record = {
|
||||
"record_type": "lid_phone_number_pair",
|
||||
"lid": lid,
|
||||
"phone_number": phone_number,
|
||||
"pair_timestamp": (
|
||||
convert_mactime_to_iso(pair_timestamp) or None
|
||||
)
|
||||
if pair_timestamp
|
||||
else None,
|
||||
}
|
||||
self.results.append(record)
|
||||
|
||||
if lid and phone_number:
|
||||
lid_digits = str(lid).split("@")[0]
|
||||
phone_digits = str(phone_number).split("@")[0].lstrip("+")
|
||||
lid_map[lid_digits] = phone_digits
|
||||
|
||||
return lid_map
|
||||
|
||||
def _extract_chat_sessions(
|
||||
self, cur: sqlite3.Cursor, lid_map: Dict[str, str]
|
||||
) -> int:
|
||||
"""Extract one record per chat session with the first and last
|
||||
interaction dates of each conversation."""
|
||||
try:
|
||||
cur.execute(CHAT_SESSIONS_QUERY)
|
||||
except sqlite3.OperationalError as exc:
|
||||
self.log.warning(
|
||||
"Unable to extract WhatsApp chat sessions: %s", exc
|
||||
)
|
||||
return 0
|
||||
|
||||
names = [description[0] for description in cur.description]
|
||||
|
||||
total_sessions = 0
|
||||
for row in cur.fetchall():
|
||||
session = dict(zip(names, row))
|
||||
session["record_type"] = "chat_session"
|
||||
|
||||
session["partner_resolved_phone_number"] = None
|
||||
jid = session.get("contact_jid") or ""
|
||||
if jid.endswith("@lid"):
|
||||
phone_digits = lid_map.get(jid.split("@")[0])
|
||||
if phone_digits:
|
||||
session["partner_resolved_phone_number"] = (
|
||||
f"+{phone_digits}"
|
||||
)
|
||||
|
||||
for field in CHAT_SESSION_DATE_FIELDS:
|
||||
if session.get(field):
|
||||
session[field] = (
|
||||
convert_mactime_to_iso(session[field]) or None
|
||||
)
|
||||
else:
|
||||
session[field] = None
|
||||
|
||||
self.results.append(session)
|
||||
total_sessions += 1
|
||||
|
||||
return total_sessions
|
||||
@@ -0,0 +1,308 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from mvt.common.module import DatabaseNotFoundError
|
||||
from mvt.common.module_types import (
|
||||
ModuleAtomicResult,
|
||||
ModuleResults,
|
||||
ModuleSerializedResult,
|
||||
)
|
||||
from mvt.common.utils import convert_mactime_to_iso
|
||||
|
||||
from ..base import IOSExtraction
|
||||
|
||||
WHATSAPP_CONTACTS_BACKUP_IDS = [
|
||||
# SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-ContactsV2.sqlite"
|
||||
"b8548dc30aa1030df0ce18ef08b882cf7ab5212f",
|
||||
]
|
||||
WHATSAPP_CONTACTS_ROOT_PATHS = [
|
||||
"private/var/mobile/Containers/Shared/AppGroup/*/ContactsV2.sqlite",
|
||||
]
|
||||
|
||||
# WhatsApp's standard disappearing-messages timer values, in seconds.
|
||||
DISAPPEARING_DURATION_LABELS = {
|
||||
86400: "24 hours",
|
||||
604800: "7 days",
|
||||
1209600: "14 days",
|
||||
2592000: "30 days",
|
||||
7776000: "90 days",
|
||||
}
|
||||
|
||||
# Output field -> candidate columns in ZWAADDRESSBOOKCONTACT, in order of
|
||||
# preference. WhatsApp renames columns across versions, so the query is built
|
||||
# from the columns actually present in the database.
|
||||
COLUMN_CANDIDATES = {
|
||||
"whatsapp_id": ["ZWHATSAPPID"],
|
||||
"lid": ["ZLID"],
|
||||
"phone_number": ["ZPHONENUMBER"],
|
||||
"localized_phone_number": ["ZLOCALIZEDPHONENUMBER"],
|
||||
"full_name": ["ZFULLNAME"],
|
||||
"given_name": ["ZGIVENNAME"],
|
||||
"last_name": ["ZLASTNAME"],
|
||||
"user_name": ["ZUSERNAME"],
|
||||
"business_name": ["ZBUSINESSNAME"],
|
||||
"about_text": ["ZABOUTTEXT"],
|
||||
"about_emoji": ["ZABOUTEMOJI"],
|
||||
"notes": ["ZNOTES"],
|
||||
"disappearing_mode_duration": ["ZDISAPPEARINGMODEDURATION"],
|
||||
"disappearing_mode_timestamp": ["ZDISAPPEARINGMODETIMESTAMP"],
|
||||
"about_timestamp": ["ZABOUTTIMESTAMP"],
|
||||
"about_expiration_timestamp": ["ZABOUTEXPIRATIONTIMESTAMP"],
|
||||
"last_updated": ["ZLASTUPDATED"],
|
||||
"phone_status": ["ZPHONESTATUS", "ZPHONENUMBERSTATUS"],
|
||||
"sync_policy": ["ZSYNCPOLICY"],
|
||||
}
|
||||
|
||||
STRING_FIELDS = [
|
||||
"whatsapp_id",
|
||||
"lid",
|
||||
"phone_number",
|
||||
"localized_phone_number",
|
||||
"full_name",
|
||||
"given_name",
|
||||
"last_name",
|
||||
"user_name",
|
||||
"business_name",
|
||||
"about_text",
|
||||
"about_emoji",
|
||||
"notes",
|
||||
]
|
||||
|
||||
DATE_FIELDS = [
|
||||
"disappearing_mode_timestamp",
|
||||
"about_timestamp",
|
||||
"about_expiration_timestamp",
|
||||
"last_updated",
|
||||
]
|
||||
|
||||
|
||||
def _decode_string(value) -> Optional[str]:
|
||||
# CoreData stores string attributes as UTF-8 blobs in some WhatsApp
|
||||
# versions, so values can arrive as either bytes or str.
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", "replace")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _label_duration(duration) -> str:
|
||||
if not duration:
|
||||
return "off"
|
||||
return DISAPPEARING_DURATION_LABELS.get(
|
||||
int(duration), f"{int(duration)} seconds"
|
||||
)
|
||||
|
||||
|
||||
def _describe_contact(record: ModuleAtomicResult) -> str:
|
||||
contact = (
|
||||
record.get("whatsapp_id")
|
||||
or record.get("lid")
|
||||
or record.get("phone_number")
|
||||
or "unknown"
|
||||
)
|
||||
full_name = record.get("full_name")
|
||||
if full_name:
|
||||
contact = f"{contact} ({full_name})"
|
||||
return contact
|
||||
|
||||
|
||||
class WhatsappContacts(IOSExtraction):
|
||||
"""This module extracts WhatsApp contact records and per-contact
|
||||
disappearing-messages settings from ContactsV2.sqlite.
|
||||
|
||||
ChatStorage.sqlite does not record the disappearing-messages state of 1:1
|
||||
chats: the authoritative timer is stored on each contact record in this
|
||||
database, alongside the mapping between a contact's LID and phone number
|
||||
identifiers.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: Optional[str] = None,
|
||||
target_path: Optional[str] = None,
|
||||
results_path: Optional[str] = None,
|
||||
module_options: Optional[dict] = None,
|
||||
log: logging.Logger = logging.getLogger(__name__),
|
||||
results: Optional[ModuleResults] = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
file_path=file_path,
|
||||
target_path=target_path,
|
||||
results_path=results_path,
|
||||
module_options=module_options,
|
||||
log=log,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||
records = []
|
||||
contact = _describe_contact(record)
|
||||
|
||||
if record.get("disappearing_mode_timestamp"):
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record["disappearing_mode_timestamp"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "disappearing_mode_set",
|
||||
"data": (
|
||||
f"WhatsApp disappearing messages timer set to "
|
||||
f"'{record.get('disappearing_mode_label')}' "
|
||||
f"for {contact}"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
if record.get("about_timestamp"):
|
||||
data = f"WhatsApp about text of {contact} changed"
|
||||
about_text = record.get("about_text")
|
||||
if about_text:
|
||||
data += f' to "{about_text}"'
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record["about_timestamp"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "about_changed",
|
||||
"data": data,
|
||||
}
|
||||
)
|
||||
|
||||
if record.get("about_expiration_timestamp"):
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record["about_expiration_timestamp"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "about_expiration",
|
||||
"data": (
|
||||
f"WhatsApp about text of {contact} scheduled "
|
||||
f"to expire"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
if record.get("last_updated"):
|
||||
records.append(
|
||||
{
|
||||
"timestamp": record["last_updated"],
|
||||
"module": self.__class__.__name__,
|
||||
"event": "contact_last_updated",
|
||||
"data": f"WhatsApp contact record for {contact} updated",
|
||||
}
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
def run(self) -> None:
|
||||
try:
|
||||
self._find_ios_database(
|
||||
backup_ids=WHATSAPP_CONTACTS_BACKUP_IDS,
|
||||
root_paths=WHATSAPP_CONTACTS_ROOT_PATHS,
|
||||
)
|
||||
except DatabaseNotFoundError:
|
||||
self.log.warning(
|
||||
"Unable to find the WhatsApp ContactsV2.sqlite database in "
|
||||
"this backup or filesystem dump. WhatsApp disappearing "
|
||||
"messages settings and contact records cannot be extracted. "
|
||||
"This database is often missing from incremental backups."
|
||||
)
|
||||
return
|
||||
|
||||
self.log.info(
|
||||
"Found WhatsApp contacts database at path: %s", self.file_path
|
||||
)
|
||||
|
||||
assert self.file_path is not None
|
||||
conn = self._open_sqlite_db(self.file_path)
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
try:
|
||||
cur.execute("PRAGMA table_info(ZWAADDRESSBOOKCONTACT)")
|
||||
available_columns = {row[1] for row in cur.fetchall()}
|
||||
except sqlite3.DatabaseError as exc:
|
||||
self.log.error(
|
||||
"Unable to read the ZWAADDRESSBOOKCONTACT table schema: %s",
|
||||
exc,
|
||||
)
|
||||
return
|
||||
|
||||
if not available_columns:
|
||||
self.log.warning(
|
||||
"The WhatsApp contacts database does not contain a "
|
||||
"ZWAADDRESSBOOKCONTACT table"
|
||||
)
|
||||
return
|
||||
|
||||
selected = {}
|
||||
for field, candidates in COLUMN_CANDIDATES.items():
|
||||
for candidate in candidates:
|
||||
if candidate in available_columns:
|
||||
selected[field] = candidate
|
||||
break
|
||||
|
||||
# A record with no duration column is "unknown", not "off": the
|
||||
# timer state cannot be determined from this database version.
|
||||
has_duration = "disappearing_mode_duration" in selected
|
||||
if not has_duration:
|
||||
self.log.warning(
|
||||
"The ZDISAPPEARINGMODEDURATION column is not present in "
|
||||
"this WhatsApp contacts database: disappearing messages "
|
||||
"state is unknown"
|
||||
)
|
||||
|
||||
columns = ["Z_PK"] + list(selected.values())
|
||||
cur.execute(
|
||||
f"SELECT {', '.join(columns)} FROM ZWAADDRESSBOOKCONTACT;"
|
||||
)
|
||||
fields = ["row_pk"] + list(selected.keys())
|
||||
|
||||
for row in cur:
|
||||
record = dict(zip(fields, row))
|
||||
|
||||
for field in STRING_FIELDS:
|
||||
if field in record:
|
||||
record[field] = _decode_string(record[field])
|
||||
else:
|
||||
record[field] = None
|
||||
|
||||
for field in DATE_FIELDS:
|
||||
if record.get(field) is not None:
|
||||
record[field] = (
|
||||
convert_mactime_to_iso(record[field]) or None
|
||||
)
|
||||
else:
|
||||
record[field] = None
|
||||
|
||||
duration = record.get("disappearing_mode_duration")
|
||||
if has_duration:
|
||||
record["disappearing_mode_is_on"] = bool(duration)
|
||||
record["disappearing_mode_label"] = _label_duration(
|
||||
duration
|
||||
)
|
||||
else:
|
||||
record["disappearing_mode_duration"] = None
|
||||
record["disappearing_mode_is_on"] = None
|
||||
record["disappearing_mode_label"] = None
|
||||
|
||||
record.setdefault("phone_status", None)
|
||||
record.setdefault("sync_policy", None)
|
||||
|
||||
self.results.append(record)
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
total_ephemeral = sum(
|
||||
1 for record in self.results if record["disappearing_mode_is_on"]
|
||||
)
|
||||
self.log.info(
|
||||
"Extracted a total of %d WhatsApp contacts (%d with disappearing "
|
||||
"messages enabled)",
|
||||
len(self.results),
|
||||
total_ephemeral,
|
||||
)
|
||||
@@ -25,6 +25,25 @@ class TestDumpsysPackagesArtifact:
|
||||
== "com.samsung.android.provider.filterprovider"
|
||||
)
|
||||
assert dpa.results[0]["version_name"] == "5.0.07"
|
||||
assert dpa.results[0]["first_install_time"] == "2008-12-31 16:00:00"
|
||||
assert dpa.results[0]["system"] is True
|
||||
|
||||
def test_parsing_system_flag(self):
|
||||
system_details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||
" pkgFlags=[ SYSTEM HAS_CODE ALLOW_CLEAR_USER_DATA ]"
|
||||
)
|
||||
third_party_details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||
" pkgFlags=[ HAS_CODE ALLOW_BACKUP ]"
|
||||
)
|
||||
missing_flag_details = (
|
||||
DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||
" versionName=1.0"
|
||||
)
|
||||
)
|
||||
|
||||
assert system_details["system"] is True
|
||||
assert third_party_details["system"] is False
|
||||
assert missing_flag_details["system"] is False
|
||||
|
||||
def test_ioc_check(self, indicator_file):
|
||||
dpa = DumpsysPackagesArtifact()
|
||||
@@ -40,3 +59,43 @@ class TestDumpsysPackagesArtifact:
|
||||
assert len(dpa.alertstore.alerts) == 0
|
||||
dpa.check_indicators()
|
||||
assert len(dpa.alertstore.alerts) == 1
|
||||
|
||||
def test_per_user_fields_use_primary_user(self):
|
||||
details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||
""" User 0: installed=true
|
||||
firstInstallTime=2024-01-10 09:19:39
|
||||
runtime permissions:
|
||||
android.permission.CAMERA: granted=true
|
||||
User 95: installed=false
|
||||
firstInstallTime=1970-01-01 01:00:00
|
||||
runtime permissions:
|
||||
android.permission.CAMERA: granted=false
|
||||
android.permission.RECORD_AUDIO: granted=false
|
||||
"""
|
||||
)
|
||||
|
||||
assert details["first_install_time"] == "2024-01-10 09:19:39"
|
||||
runtime_permissions = [
|
||||
permission
|
||||
for permission in details["permissions"]
|
||||
if permission["type"] == "runtime"
|
||||
]
|
||||
assert runtime_permissions == [
|
||||
{
|
||||
"name": "android.permission.CAMERA",
|
||||
"granted": True,
|
||||
"type": "runtime",
|
||||
}
|
||||
]
|
||||
|
||||
def test_per_user_fields_fall_back_when_user_zero_is_missing(self):
|
||||
details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||
""" User 10: installed=true
|
||||
firstInstallTime=2024-02-10 09:19:39
|
||||
runtime permissions:
|
||||
android.permission.CAMERA: granted=true
|
||||
"""
|
||||
)
|
||||
|
||||
assert details["first_install_time"] == "2024-02-10 09:19:39"
|
||||
assert details["permissions"][-1]["name"] == "android.permission.CAMERA"
|
||||
@@ -8,6 +8,7 @@ import datetime
|
||||
import pytest
|
||||
|
||||
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
|
||||
from mvt.android.parsers.proto.tombstone import Tombstone
|
||||
|
||||
from ..utils import get_artifact
|
||||
|
||||
@@ -42,6 +43,47 @@ class TestTombstoneCrashArtifact:
|
||||
assert len(tombstone_artifact.results) == 1
|
||||
self.validate_tombstone_result(tombstone_artifact.results[0])
|
||||
|
||||
def test_text_tombstone_preserves_abort_message(self):
|
||||
tombstone_artifact = TombstoneCrashArtifact()
|
||||
artifact_path = "android_data/bugreport/FS/data/tombstones/tombstone_00"
|
||||
file = get_artifact(artifact_path)
|
||||
with open(file, "rb") as f:
|
||||
data = f.read()
|
||||
|
||||
tombstone_artifact.parse(
|
||||
os.path.basename(artifact_path),
|
||||
datetime.datetime(2021, 9, 29, 17, 43, 49),
|
||||
data,
|
||||
)
|
||||
|
||||
assert tombstone_artifact.results[0]["abort_message"] == (
|
||||
"Check failed: payload.size() <= bytes_left "
|
||||
"(payload.size()=99, bytes_left=51) "
|
||||
)
|
||||
|
||||
def test_protobuf_tombstone_preserves_abort_message_and_causes(self):
|
||||
tombstone_artifact = TombstoneCrashArtifact()
|
||||
artifact_path = "android_data/tombstone_process.pb"
|
||||
file = get_artifact(artifact_path)
|
||||
with open(file, "rb") as f:
|
||||
tombstone = Tombstone().parse(f.read())
|
||||
|
||||
tombstone.abort_message = "synthetic abort reason"
|
||||
tombstone_artifact.parse_protobuf(
|
||||
os.path.basename(artifact_path),
|
||||
datetime.datetime(2023, 4, 12, 12, 32, 40, 518290),
|
||||
bytes(tombstone),
|
||||
)
|
||||
|
||||
result = tombstone_artifact.results[0]
|
||||
assert result["abort_message"] == "synthetic abort reason"
|
||||
assert result["causes"] == [
|
||||
{
|
||||
"human_readable": "null pointer dereference",
|
||||
"memory_error": None,
|
||||
}
|
||||
]
|
||||
|
||||
def test_text_tombstone_keeps_crashing_thread(self):
|
||||
tombstone_artifact = TombstoneCrashArtifact()
|
||||
artifact_path = "android_data/tombstone_process.txt"
|
||||
|
||||
@@ -54,6 +54,8 @@ class TestBugreportAnalysis:
|
||||
== "com.samsung.android.provider.filterprovider"
|
||||
)
|
||||
assert m.results[1]["package_name"] == "com.instagram.android"
|
||||
assert m.results[0]["installer"] == ""
|
||||
assert m.results[1]["installer"] == "com.android.vending"
|
||||
assert len(m.results[0]["permissions"]) == 4
|
||||
assert len(m.results[1]["permissions"]) == 32
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,565 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import click
|
||||
from click.testing import CliRunner
|
||||
|
||||
from mvt.cli import cli as mvt_cli
|
||||
from mvt.common.cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
MVT_CUSTOM_COMMANDS_ENV,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
BrokenPluginCommand,
|
||||
load_cli_commands_option,
|
||||
register_cli_commands_from_path,
|
||||
register_cli_plugins,
|
||||
register_installed_cli_commands,
|
||||
)
|
||||
|
||||
|
||||
# Keep the banner of the mvt group callback from checking for updates online.
|
||||
OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"]
|
||||
|
||||
COMMAND_TEMPLATE = """
|
||||
import click
|
||||
|
||||
|
||||
@click.command({name!r})
|
||||
@click.pass_context
|
||||
def cli(ctx):
|
||||
click.echo({message!r})
|
||||
if ctx.obj:
|
||||
click.echo(ctx.obj.get("marker", ""))
|
||||
"""
|
||||
|
||||
|
||||
def _write_command(path, name, message="command ran"):
|
||||
path.write_text(
|
||||
COMMAND_TEMPLATE.format(name=name, message=message),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _make_group():
|
||||
@click.group()
|
||||
@load_cli_commands_option
|
||||
@click.pass_context
|
||||
def group(ctx):
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["marker"] = "parent context"
|
||||
|
||||
return group
|
||||
|
||||
|
||||
def _entry_point(name, value, command=None, exception=None, distribution="plugin"):
|
||||
def load():
|
||||
if exception is not None:
|
||||
raise exception
|
||||
return command
|
||||
|
||||
dist = SimpleNamespace(metadata={"Name": distribution}, version="1.0")
|
||||
return SimpleNamespace(name=name, value=value, load=load, dist=dist)
|
||||
|
||||
|
||||
def test_load_command_option_registers_command_before_resolution(tmp_path):
|
||||
command_path = _write_command(tmp_path / "hello.py", "hello")
|
||||
group = _make_group()
|
||||
|
||||
result = CliRunner().invoke(
|
||||
group,
|
||||
["--load-command", str(command_path), "hello"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "command ran" in result.output
|
||||
assert "parent context" in result.output
|
||||
|
||||
|
||||
def test_load_command_option_supports_folders_and_repeated_paths(tmp_path):
|
||||
folder = tmp_path / "commands"
|
||||
folder.mkdir()
|
||||
_write_command(folder / "b.py", "second")
|
||||
_write_command(folder / "a.py", "first")
|
||||
_write_command(folder / ".hidden.py", "hidden")
|
||||
_write_command(folder / "__init__.py", "init")
|
||||
other = _write_command(tmp_path / "third.py", "third")
|
||||
group = _make_group()
|
||||
|
||||
result = CliRunner().invoke(
|
||||
group,
|
||||
[
|
||||
"--load-command",
|
||||
str(folder),
|
||||
"--load-command",
|
||||
str(other),
|
||||
"--load-command",
|
||||
str(other),
|
||||
"--help",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "first" in result.output
|
||||
assert "second" in result.output
|
||||
assert "third" in result.output
|
||||
assert "hidden" not in result.output
|
||||
assert "init" not in result.output
|
||||
|
||||
|
||||
def test_loaded_command_participates_in_shell_completion(tmp_path):
|
||||
command_path = _write_command(tmp_path / "hello.py", "hello")
|
||||
group = _make_group()
|
||||
words = f"group --load-command {command_path} he"
|
||||
|
||||
result = CliRunner().invoke(
|
||||
group,
|
||||
[],
|
||||
env={
|
||||
"_GROUP_COMPLETE": "bash_complete",
|
||||
"COMP_WORDS": words,
|
||||
"COMP_CWORD": "3",
|
||||
},
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "plain,hello" in result.output
|
||||
|
||||
|
||||
def test_explicit_command_import_and_contract_failures_are_usage_errors(tmp_path):
|
||||
broken_path = tmp_path / "broken.py"
|
||||
broken_path.write_text("raise RuntimeError('broken import')", encoding="utf-8")
|
||||
missing_cli_path = tmp_path / "missing_cli.py"
|
||||
missing_cli_path.write_text("value = 1", encoding="utf-8")
|
||||
|
||||
broken_result = CliRunner().invoke(
|
||||
_make_group(),
|
||||
["--load-command", str(broken_path), "broken"],
|
||||
)
|
||||
missing_cli_result = CliRunner().invoke(
|
||||
_make_group(),
|
||||
["--load-command", str(missing_cli_path), "missing-cli"],
|
||||
)
|
||||
|
||||
assert broken_result.exit_code == 2
|
||||
assert "broken import" in broken_result.output
|
||||
assert missing_cli_result.exit_code == 2
|
||||
assert "must export a Click command or group named 'cli'" in (
|
||||
missing_cli_result.output
|
||||
)
|
||||
|
||||
|
||||
def test_environment_command_failure_gets_broken_placeholder(tmp_path):
|
||||
command_path = tmp_path / "broken_command.py"
|
||||
command_path.write_text("raise RuntimeError('broken import')", encoding="utf-8")
|
||||
group = click.Group()
|
||||
|
||||
registered = register_cli_commands_from_path(group, command_path)
|
||||
|
||||
assert registered == ["broken-command"]
|
||||
assert isinstance(group.commands["broken-command"], BrokenPluginCommand)
|
||||
result = CliRunner().invoke(group, ["broken-command"])
|
||||
assert result.exit_code == 1
|
||||
assert "broken import" in result.output
|
||||
assert str(command_path) in result.output
|
||||
|
||||
|
||||
def test_environment_command_system_exit_gets_broken_placeholder(tmp_path):
|
||||
command_path = tmp_path / "exiting_command.py"
|
||||
command_path.write_text("raise SystemExit(7)", encoding="utf-8")
|
||||
group = click.Group()
|
||||
|
||||
registered = register_cli_commands_from_path(group, command_path)
|
||||
|
||||
assert registered == ["exiting-command"]
|
||||
assert isinstance(group.commands["exiting-command"], BrokenPluginCommand)
|
||||
result = CliRunner().invoke(group, ["exiting-command"])
|
||||
assert result.exit_code == 1
|
||||
assert "Unable to import custom command" in result.output
|
||||
assert result.output.rstrip().endswith(": 7")
|
||||
|
||||
|
||||
def test_installed_entry_point_name_is_the_command_name(monkeypatch):
|
||||
@click.command("internal-name")
|
||||
def command():
|
||||
click.echo("installed command ran")
|
||||
|
||||
entry_point = _entry_point(
|
||||
"external-name",
|
||||
"example_plugin:cli",
|
||||
command=command,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
lambda **kwargs: [entry_point],
|
||||
)
|
||||
group = click.Group()
|
||||
|
||||
registered = register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||
|
||||
assert registered == ["external-name"]
|
||||
assert "internal-name" not in group.commands
|
||||
result = CliRunner().invoke(group, ["external-name"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output == "installed command ran\n"
|
||||
|
||||
|
||||
def test_broken_installed_plugin_does_not_break_cli(monkeypatch):
|
||||
broken = _entry_point(
|
||||
"broken",
|
||||
"broken_plugin:cli",
|
||||
exception=RuntimeError("missing dependency"),
|
||||
distribution="broken-plugin",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
lambda **kwargs: [broken],
|
||||
)
|
||||
group = click.Group()
|
||||
|
||||
register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||
|
||||
help_result = CliRunner().invoke(group, ["--help"])
|
||||
assert help_result.exit_code == 0
|
||||
assert "Warning: external command could not be loaded." in help_result.output
|
||||
|
||||
result = CliRunner().invoke(group, ["broken"])
|
||||
assert result.exit_code == 1
|
||||
assert "broken-plugin 1.0 (broken_plugin:cli)" in result.output
|
||||
assert "RuntimeError: missing dependency" in result.output
|
||||
|
||||
|
||||
def test_installed_plugin_system_exit_does_not_break_cli(monkeypatch):
|
||||
exiting = _entry_point(
|
||||
"exiting",
|
||||
"exiting_plugin:cli",
|
||||
exception=SystemExit(7),
|
||||
distribution="exiting-plugin",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
lambda **kwargs: [exiting],
|
||||
)
|
||||
group = click.Group()
|
||||
|
||||
register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||
|
||||
help_result = CliRunner().invoke(group, ["--help"])
|
||||
assert help_result.exit_code == 0
|
||||
result = CliRunner().invoke(group, ["exiting"])
|
||||
assert result.exit_code == 1
|
||||
assert "SystemExit: 7" in result.output
|
||||
|
||||
|
||||
def test_non_click_entry_point_gets_broken_placeholder(monkeypatch):
|
||||
invalid = _entry_point("invalid", "plugin:value", command=object())
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
lambda **kwargs: [invalid],
|
||||
)
|
||||
group = click.Group()
|
||||
|
||||
register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||
|
||||
assert isinstance(group.commands["invalid"], BrokenPluginCommand)
|
||||
result = CliRunner().invoke(group, ["invalid"])
|
||||
assert result.exit_code == 1
|
||||
assert "must resolve to a Click command or group" in result.output
|
||||
|
||||
|
||||
def test_entry_point_discovery_failure_does_not_break_group(monkeypatch, caplog):
|
||||
def fail_discovery(**kwargs):
|
||||
raise RuntimeError("invalid package metadata")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
fail_discovery,
|
||||
)
|
||||
group = click.Group()
|
||||
|
||||
registered = register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||
|
||||
assert registered == []
|
||||
assert not group.commands
|
||||
assert "Unable to discover external commands" in caplog.text
|
||||
assert "invalid package metadata" in caplog.text
|
||||
|
||||
|
||||
def test_builtin_and_first_external_command_win_collisions(monkeypatch, caplog):
|
||||
@click.command("version")
|
||||
def core_version():
|
||||
pass
|
||||
|
||||
@click.command()
|
||||
def first():
|
||||
pass
|
||||
|
||||
@click.command()
|
||||
def second():
|
||||
pass
|
||||
|
||||
entry_points = [
|
||||
_entry_point("duplicate", "z_plugin:cli", command=second, distribution="z"),
|
||||
_entry_point("version", "plugin:version", command=first),
|
||||
_entry_point("duplicate", "a_plugin:cli", command=first, distribution="a"),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
lambda **kwargs: entry_points,
|
||||
)
|
||||
group = click.Group(commands={"version": core_version})
|
||||
|
||||
registered = register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||
|
||||
assert registered == ["duplicate"]
|
||||
assert group.commands["version"] is core_version
|
||||
assert group.commands["duplicate"] is first
|
||||
assert "the command name is already registered" in caplog.text
|
||||
|
||||
|
||||
def test_explicit_command_cannot_replace_existing_command(tmp_path):
|
||||
command_path = _write_command(tmp_path / "version.py", "version")
|
||||
group = _make_group()
|
||||
|
||||
@group.command("version")
|
||||
def core_version():
|
||||
pass
|
||||
|
||||
result = CliRunner().invoke(
|
||||
group,
|
||||
["--load-command", str(command_path), "version"],
|
||||
)
|
||||
|
||||
assert result.exit_code == 2
|
||||
assert "the command name is already registered" in result.output
|
||||
|
||||
|
||||
def test_platform_entry_point_groups_and_environment_paths_are_separate(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
ios_path = _write_command(tmp_path / "ios.py", "ios-file")
|
||||
android_path = _write_command(tmp_path / "android.py", "android-file")
|
||||
|
||||
@click.command()
|
||||
def ios_package():
|
||||
pass
|
||||
|
||||
@click.command()
|
||||
def android_package():
|
||||
pass
|
||||
|
||||
def entry_points(*, group):
|
||||
if group == IOS_CLI_PLUGIN_GROUP:
|
||||
return [_entry_point("ios-package", "ios_plugin:cli", ios_package)]
|
||||
if group == ANDROID_CLI_PLUGIN_GROUP:
|
||||
return [
|
||||
_entry_point("android-package", "android_plugin:cli", android_package)
|
||||
]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
entry_points,
|
||||
)
|
||||
monkeypatch.setenv("TEST_IOS_COMMANDS", str(ios_path))
|
||||
monkeypatch.setenv("TEST_ANDROID_COMMANDS", str(android_path))
|
||||
ios_group = click.Group()
|
||||
android_group = click.Group()
|
||||
|
||||
register_cli_plugins(
|
||||
ios_group,
|
||||
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
||||
environment_variable="TEST_IOS_COMMANDS",
|
||||
)
|
||||
register_cli_plugins(
|
||||
android_group,
|
||||
entry_point_group=ANDROID_CLI_PLUGIN_GROUP,
|
||||
environment_variable="TEST_ANDROID_COMMANDS",
|
||||
)
|
||||
|
||||
assert set(ios_group.commands) == {"ios-file", "ios-package"}
|
||||
assert set(android_group.commands) == {"android-file", "android-package"}
|
||||
|
||||
|
||||
def test_neutral_entry_point_group_is_not_registered_on_the_platform_clis(
|
||||
monkeypatch,
|
||||
):
|
||||
@click.command()
|
||||
def neutral_package():
|
||||
pass
|
||||
|
||||
def entry_points(*, group):
|
||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
||||
return [
|
||||
_entry_point("neutral-package", "neutral_plugin:cli", neutral_package)
|
||||
]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
entry_points,
|
||||
)
|
||||
ios_group = click.Group()
|
||||
android_group = click.Group()
|
||||
|
||||
register_cli_plugins(
|
||||
ios_group,
|
||||
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
||||
environment_variable="TEST_IOS_COMMANDS",
|
||||
)
|
||||
register_cli_plugins(
|
||||
android_group,
|
||||
entry_point_group=ANDROID_CLI_PLUGIN_GROUP,
|
||||
environment_variable="TEST_ANDROID_COMMANDS",
|
||||
)
|
||||
|
||||
assert not ios_group.commands
|
||||
assert not android_group.commands
|
||||
|
||||
|
||||
def test_environment_command_wins_collision_with_installed_command(
|
||||
tmp_path, monkeypatch, caplog
|
||||
):
|
||||
command_path = _write_command(
|
||||
tmp_path / "duplicate.py",
|
||||
"duplicate",
|
||||
message="environment command ran",
|
||||
)
|
||||
|
||||
@click.command()
|
||||
def installed_command():
|
||||
pass
|
||||
|
||||
def entry_points(*, group):
|
||||
if group == IOS_CLI_PLUGIN_GROUP:
|
||||
return [
|
||||
_entry_point(
|
||||
"duplicate",
|
||||
"ios_plugin:cli",
|
||||
installed_command,
|
||||
distribution="ios-plugin",
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
entry_points,
|
||||
)
|
||||
monkeypatch.setenv("TEST_IOS_COMMANDS", str(command_path))
|
||||
group = click.Group()
|
||||
|
||||
register_cli_plugins(
|
||||
group,
|
||||
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
||||
environment_variable="TEST_IOS_COMMANDS",
|
||||
)
|
||||
|
||||
assert group.commands["duplicate"] is not installed_command
|
||||
result = CliRunner().invoke(group, ["duplicate"])
|
||||
assert result.exit_code == 0
|
||||
assert "environment command ran" in result.output
|
||||
assert "the command name is already registered" in caplog.text
|
||||
assert "ios-plugin 1.0 (ios_plugin:cli)" in caplog.text
|
||||
|
||||
|
||||
def test_the_mvt_cli_gets_the_neutral_commands_and_no_platform_command(
|
||||
monkeypatch, restore_cli_commands
|
||||
):
|
||||
@click.command()
|
||||
def shared_package():
|
||||
click.echo("shared command ran")
|
||||
|
||||
@click.command()
|
||||
def ios_package():
|
||||
pass
|
||||
|
||||
def entry_points(*, group):
|
||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
||||
return [_entry_point("shared-package", "shared_plugin:cli", shared_package)]
|
||||
if group == IOS_CLI_PLUGIN_GROUP:
|
||||
return [_entry_point("ios-package", "ios_plugin:cli", ios_package)]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
entry_points,
|
||||
)
|
||||
|
||||
register_cli_plugins(
|
||||
mvt_cli,
|
||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
|
||||
assert "ios-package" not in mvt_cli.commands
|
||||
result = CliRunner().invoke(mvt_cli, [*OFFLINE, "shared-package"])
|
||||
assert result.exit_code == 0
|
||||
assert "shared command ran" in result.output
|
||||
|
||||
|
||||
def test_builtin_mvt_command_wins_collision_with_neutral_command(
|
||||
monkeypatch, caplog, restore_cli_commands
|
||||
):
|
||||
@click.command()
|
||||
def neutral_version():
|
||||
pass
|
||||
|
||||
def entry_points(*, group):
|
||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
||||
return [
|
||||
_entry_point(
|
||||
"version",
|
||||
"neutral_plugin:cli",
|
||||
neutral_version,
|
||||
distribution="neutral-plugin",
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
entry_points,
|
||||
)
|
||||
builtin_version = mvt_cli.commands["version"]
|
||||
|
||||
register_cli_plugins(
|
||||
mvt_cli,
|
||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
|
||||
assert mvt_cli.commands["version"] is builtin_version
|
||||
assert "the command name is already registered" in caplog.text
|
||||
assert "neutral-plugin 1.0 (neutral_plugin:cli)" in caplog.text
|
||||
|
||||
|
||||
def test_broken_neutral_plugin_does_not_break_the_mvt_cli(
|
||||
monkeypatch, restore_cli_commands
|
||||
):
|
||||
def entry_points(*, group):
|
||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
||||
return [
|
||||
_entry_point(
|
||||
"broken",
|
||||
"broken_plugin:cli",
|
||||
exception=RuntimeError("missing dependency"),
|
||||
distribution="broken-plugin",
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||
entry_points,
|
||||
)
|
||||
|
||||
register_cli_plugins(
|
||||
mvt_cli,
|
||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
|
||||
assert isinstance(mvt_cli.commands["broken"], BrokenPluginCommand)
|
||||
result = CliRunner().invoke(mvt_cli, [*OFFLINE, "version"])
|
||||
assert result.exit_code == 0
|
||||
@@ -0,0 +1,236 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from mvt.android.cli import cli as android_cli
|
||||
from mvt.cli import cli as mvt_cli
|
||||
from mvt.common.cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
from mvt.common.cmd_plugins import plugins
|
||||
from mvt.common.module import MVTModule
|
||||
from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP
|
||||
from mvt.common.updates import PluginUpdates
|
||||
from mvt.ios.cli import cli as ios_cli
|
||||
|
||||
|
||||
class ExampleModule(MVTModule):
|
||||
pass
|
||||
|
||||
|
||||
class AnotherModule(MVTModule):
|
||||
pass
|
||||
|
||||
|
||||
class FakeDistribution:
|
||||
def __init__(self, name, version="1.0.0", direct_url=None):
|
||||
self.name = name
|
||||
self.version = version
|
||||
self.direct_url = direct_url
|
||||
|
||||
def read_text(self, file_name):
|
||||
if file_name == "direct_url.json" and self.direct_url is not None:
|
||||
return json.dumps(self.direct_url)
|
||||
return None
|
||||
|
||||
|
||||
def _entry_point(name, distribution, modules=None, exception=None):
|
||||
def load():
|
||||
if exception is not None:
|
||||
raise exception
|
||||
return modules
|
||||
|
||||
return SimpleNamespace(
|
||||
name=name, value="example_plugin:modules", dist=distribution, load=load
|
||||
)
|
||||
|
||||
|
||||
def _run(command, arguments):
|
||||
# Keep rich from wrapping the table while its content is being asserted.
|
||||
return CliRunner().invoke(command, arguments, env={"COLUMNS": "200"})
|
||||
|
||||
|
||||
def _table_rows(output):
|
||||
"""Return the content of the table rows, without the header and the box."""
|
||||
return [
|
||||
[cell.strip() for cell in line.strip().strip("│").split("│")]
|
||||
for line in output.splitlines()
|
||||
if "│" in line
|
||||
]
|
||||
|
||||
|
||||
def _table_header(output):
|
||||
for line in output.splitlines():
|
||||
if "┃" in line:
|
||||
return [cell.strip() for cell in line.strip().strip("┃").split("┃")]
|
||||
return []
|
||||
|
||||
|
||||
def _install(monkeypatch, distributions, entry_points):
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cmd_plugins.installed_plugin_distributions",
|
||||
lambda: distributions,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cmd_plugins.importlib.metadata.entry_points",
|
||||
lambda *, group: entry_points.get(group, []),
|
||||
)
|
||||
|
||||
|
||||
def test_plugins_is_a_builtin_command_of_the_mvt_cli_only():
|
||||
assert mvt_cli.commands["plugins"] is plugins
|
||||
assert "plugins" not in ios_cli.commands
|
||||
assert "plugins" not in android_cli.commands
|
||||
|
||||
|
||||
def test_list_shows_what_every_plugin_contributes(monkeypatch):
|
||||
index_plugin = FakeDistribution("example-plugin", version="1.2.0")
|
||||
repository_plugin = FakeDistribution(
|
||||
"repository-plugin",
|
||||
version="0.1.0",
|
||||
direct_url={
|
||||
"url": "https://example.org/plugin.git",
|
||||
"vcs_info": {"vcs": "git", "commit_id": "b" * 40},
|
||||
},
|
||||
)
|
||||
local_plugin = FakeDistribution(
|
||||
"local-plugin",
|
||||
direct_url={"url": "file:///plugins", "dir_info": {"editable": True}},
|
||||
)
|
||||
_install(
|
||||
monkeypatch,
|
||||
[index_plugin, local_plugin, repository_plugin],
|
||||
{
|
||||
MODULES_ENTRY_POINT_GROUP: [
|
||||
_entry_point(
|
||||
"example", index_plugin, modules=[ExampleModule, AnotherModule]
|
||||
),
|
||||
_entry_point("local", local_plugin, modules=lambda: [ExampleModule]),
|
||||
],
|
||||
IOS_CLI_PLUGIN_GROUP: [_entry_point("summarize", repository_plugin)],
|
||||
ANDROID_CLI_PLUGIN_GROUP: [_entry_point("triage", local_plugin)],
|
||||
NEUTRAL_CLI_PLUGIN_GROUP: [_entry_point("report", repository_plugin)],
|
||||
},
|
||||
)
|
||||
|
||||
result = _run(plugins, ["list"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
# Plugins are listed by name, with the modules and the commands each of
|
||||
# them contributes.
|
||||
assert _table_header(result.output) == [
|
||||
"Name",
|
||||
"Version",
|
||||
"Origin",
|
||||
"Modules",
|
||||
"Commands",
|
||||
]
|
||||
assert _table_rows(result.output) == [
|
||||
["example-plugin", "1.2.0", "pypi", "2", "-"],
|
||||
["local-plugin", "1.0.0", "local", "1", "triage"],
|
||||
["repository-plugin", "0.1.0", "git+bbbbbbbb", "0", "report, summarize"],
|
||||
]
|
||||
|
||||
|
||||
def test_list_reports_a_broken_module_entry_point(monkeypatch):
|
||||
plugin = FakeDistribution("broken-plugin")
|
||||
_install(
|
||||
monkeypatch,
|
||||
[plugin],
|
||||
{
|
||||
MODULES_ENTRY_POINT_GROUP: [
|
||||
_entry_point(
|
||||
"broken", plugin, exception=ImportError("missing dependency")
|
||||
)
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
result = _run(plugins, ["list"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert _table_rows(result.output) == [
|
||||
["broken-plugin", "1.0.0", "pypi", "error", "-"]
|
||||
]
|
||||
|
||||
|
||||
def test_list_without_plugins(monkeypatch):
|
||||
_install(monkeypatch, [], {})
|
||||
|
||||
result = _run(plugins, ["list"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert result.output == "No MVT plugins are installed.\n"
|
||||
|
||||
|
||||
def test_check_updates_prints_the_findings_and_ignores_the_throttle(monkeypatch):
|
||||
findings = [
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
}
|
||||
]
|
||||
_install(monkeypatch, [FakeDistribution("example-plugin")], {})
|
||||
monkeypatch.setattr(PluginUpdates, "check", lambda self: findings)
|
||||
monkeypatch.setattr(
|
||||
PluginUpdates,
|
||||
"should_check",
|
||||
lambda self: pytest.fail("an explicit check must not be throttled"),
|
||||
)
|
||||
|
||||
result = _run(plugins, ["check-updates"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Plugin updates available:" in result.output
|
||||
assert "example-plugin 1.0.0 → 1.2.0" in result.output
|
||||
assert "Upgrade with: pip install -U example-plugin" in result.output
|
||||
assert "MVT does not install plugin updates." in result.output
|
||||
|
||||
|
||||
def test_check_updates_without_available_updates(monkeypatch):
|
||||
_install(monkeypatch, [FakeDistribution("example-plugin")], {})
|
||||
monkeypatch.setattr(PluginUpdates, "check", lambda self: [])
|
||||
|
||||
result = _run(plugins, ["check-updates"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "All plugins are up to date." in result.output
|
||||
|
||||
|
||||
def test_check_updates_without_plugins(monkeypatch):
|
||||
_install(monkeypatch, [], {})
|
||||
monkeypatch.setattr(
|
||||
PluginUpdates,
|
||||
"check",
|
||||
lambda self: pytest.fail("nothing must be checked without plugins"),
|
||||
)
|
||||
|
||||
result = _run(plugins, ["check-updates"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "No MVT plugins are installed." in result.output
|
||||
|
||||
|
||||
def test_check_updates_without_network_access(monkeypatch):
|
||||
monkeypatch.setattr("mvt.common.cmd_plugins.settings.NETWORK_ACCESS_ALLOWED", False)
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cmd_plugins.installed_plugin_distributions",
|
||||
lambda: pytest.fail("plugins must not be listed without network access"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
PluginUpdates,
|
||||
"check",
|
||||
lambda self: pytest.fail("nothing must be checked without network access"),
|
||||
)
|
||||
|
||||
result = _run(plugins, ["check-updates"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Network access is disabled" in result.output
|
||||
@@ -157,7 +157,7 @@ class TestCommand:
|
||||
assert not hasattr(cmd, "initialized")
|
||||
assert "Circular module dependency detected" in caplog.text
|
||||
|
||||
def test_unavailable_dependency_warns_and_stops(self, caplog):
|
||||
def test_unavailable_dependency_only_skips_the_dependent_module(self, caplog):
|
||||
class UnavailableModule(RecordingModule):
|
||||
pass
|
||||
|
||||
@@ -165,14 +165,97 @@ class TestCommand:
|
||||
dependencies = (UnavailableModule,)
|
||||
|
||||
cmd = RecordingCommand()
|
||||
cmd.modules = [DependentModule]
|
||||
cmd.modules = [DependentModule, IndependentModule, FirstModule]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
cmd.run()
|
||||
|
||||
assert RecordingModule.run_order == ["IndependentModule", "FirstModule"]
|
||||
assert cmd.initialized
|
||||
assert "Module DependentModule will be SKIPPED" in caplog.text
|
||||
assert "depends on module UnavailableModule" in caplog.text
|
||||
|
||||
def test_modules_depending_on_a_skipped_module_are_skipped_too(self, caplog):
|
||||
class UnavailableModule(RecordingModule):
|
||||
pass
|
||||
|
||||
class SkippedModule(RecordingModule):
|
||||
dependencies = (UnavailableModule,)
|
||||
|
||||
class DependsOnSkippedModule(RecordingModule):
|
||||
dependencies = (SkippedModule,)
|
||||
|
||||
class DependsOnTheChain(RecordingModule):
|
||||
dependencies = (DependsOnSkippedModule,)
|
||||
|
||||
cmd = RecordingCommand()
|
||||
cmd.modules = [
|
||||
DependsOnTheChain,
|
||||
DependsOnSkippedModule,
|
||||
SkippedModule,
|
||||
IndependentModule,
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
cmd.run()
|
||||
|
||||
assert RecordingModule.run_order == ["IndependentModule"]
|
||||
skip_warnings = [
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if "will be SKIPPED" in record.getMessage()
|
||||
]
|
||||
assert len(skip_warnings) == 3
|
||||
assert [warning.split()[1] for warning in skip_warnings] == [
|
||||
"SkippedModule",
|
||||
"DependsOnSkippedModule",
|
||||
"DependsOnTheChain",
|
||||
]
|
||||
# Every warning names the root cause: the module missing a dependency
|
||||
# and the dependency it is missing.
|
||||
assert all("UnavailableModule" in warning for warning in skip_warnings)
|
||||
assert all("module SkippedModule" in warning for warning in skip_warnings[1:])
|
||||
|
||||
def test_explicitly_selected_module_with_missing_dependency_runs_nothing(
|
||||
self, caplog
|
||||
):
|
||||
class UnavailableModule(RecordingModule):
|
||||
pass
|
||||
|
||||
class DependentModule(RecordingModule):
|
||||
dependencies = (UnavailableModule,)
|
||||
|
||||
cmd = RecordingCommand(module_name="DependentModule")
|
||||
cmd.modules = [DependentModule, IndependentModule]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
cmd.run()
|
||||
|
||||
assert RecordingModule.run_order == []
|
||||
assert not hasattr(cmd, "initialized")
|
||||
assert "depends on unavailable module UnavailableModule" in caplog.text
|
||||
assert "Module DependentModule will be SKIPPED" in caplog.text
|
||||
assert "No modules will be run" in caplog.text
|
||||
# Nothing else was selected, so the warning must not promise that the
|
||||
# analysis continues right before saying that it does not.
|
||||
assert "The rest of the analysis will still run" not in caplog.text
|
||||
|
||||
def test_unaffected_dependency_chains_keep_their_order(self, caplog):
|
||||
class UnavailableModule(RecordingModule):
|
||||
pass
|
||||
|
||||
class SkippedModule(RecordingModule):
|
||||
dependencies = (UnavailableModule,)
|
||||
|
||||
cmd = RecordingCommand()
|
||||
cmd.modules = [ThirdModule, SkippedModule, SecondModule, FirstModule]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
cmd.run()
|
||||
|
||||
assert RecordingModule.run_order == [
|
||||
"FirstModule",
|
||||
"SecondModule",
|
||||
"ThirdModule",
|
||||
]
|
||||
|
||||
def test_custom_modules_are_filtered_before_ordering(self):
|
||||
cmd = RecordingCommand()
|
||||
|
||||
@@ -3,10 +3,12 @@ import pytest
|
||||
from mvt.common.module import MVTModule
|
||||
from mvt.common.module_loader import (
|
||||
CustomModuleLoadError,
|
||||
get_module_logger,
|
||||
load_custom_modules,
|
||||
load_custom_modules_from_path,
|
||||
module_supports_command,
|
||||
)
|
||||
from mvt.ios.modules.mixed.whatsapp import Whatsapp
|
||||
|
||||
|
||||
MODULE_TEMPLATE = """
|
||||
@@ -144,3 +146,44 @@ def test_module_supports_command_honors_supported_commands(tmp_path):
|
||||
|
||||
assert module_supports_command(module, "ios", "check-backup")
|
||||
assert not module_supports_command(module, "ios", "check-fs")
|
||||
|
||||
|
||||
def test_get_module_logger_keeps_builtin_names():
|
||||
assert get_module_logger(Whatsapp).name == "mvt.ios.modules.mixed.whatsapp"
|
||||
|
||||
|
||||
def test_get_module_logger_parents_package_modules_under_mvt_ext():
|
||||
class PackageModule(MVTModule):
|
||||
pass
|
||||
|
||||
PackageModule.__module__ = "some_plugin_package.ios.custom"
|
||||
|
||||
assert (
|
||||
get_module_logger(PackageModule).name
|
||||
== "mvt.ext.some_plugin_package.ios.custom"
|
||||
)
|
||||
|
||||
|
||||
def test_get_module_logger_strips_the_plugin_package_prefix():
|
||||
class PluginModule(MVTModule):
|
||||
pass
|
||||
|
||||
PluginModule.__module__ = "mvt_plugin_amnesty_custom.ios.custom"
|
||||
|
||||
assert get_module_logger(PluginModule).name == "mvt.ext.amnesty_custom.ios.custom"
|
||||
|
||||
|
||||
def test_get_module_logger_only_strips_the_prefix_from_the_top_level():
|
||||
class NestedModule(MVTModule):
|
||||
pass
|
||||
|
||||
NestedModule.__module__ = "other_package.mvt_plugin_sub"
|
||||
|
||||
assert get_module_logger(NestedModule).name == "mvt.ext.other_package.mvt_plugin_sub"
|
||||
|
||||
|
||||
def test_get_module_logger_names_path_modules_after_their_file(tmp_path):
|
||||
module_path = _write_module(tmp_path / "my_custom_module.py", "PathModule")
|
||||
module = load_custom_modules_from_path(str(module_path))[0]
|
||||
|
||||
assert get_module_logger(module).name == "mvt.ext.my_custom_module"
|
||||
@@ -0,0 +1,391 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from mvt.common.plugin_config import (
|
||||
MVTPluginSettings,
|
||||
PluginConfigLoadError,
|
||||
plugin_config_folder,
|
||||
plugin_config_path,
|
||||
plugin_data_folder,
|
||||
plugin_env_prefix,
|
||||
)
|
||||
|
||||
|
||||
class ExamplePluginSettings(MVTPluginSettings):
|
||||
plugin_name = "example-plugin"
|
||||
|
||||
API_KEY: Optional[str] = None
|
||||
CACHE_FOLDER: str = "cache"
|
||||
MAX_RESULTS: int = 25
|
||||
|
||||
|
||||
class OtherPluginSettings(MVTPluginSettings):
|
||||
plugin_name = "other-plugin"
|
||||
|
||||
API_KEY: Optional[str] = None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config_folder(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.plugin_config.user_config_dir",
|
||||
lambda *args, **kwargs: str(tmp_path),
|
||||
)
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def data_folder(tmp_path, monkeypatch):
|
||||
folder = tmp_path / "data"
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.plugin_config.user_data_dir",
|
||||
lambda *args, **kwargs: str(folder),
|
||||
)
|
||||
return folder
|
||||
|
||||
|
||||
def _write_plugin_file(plugin_name, values):
|
||||
config_path = plugin_config_path(plugin_name)
|
||||
os.makedirs(os.path.dirname(config_path), exist_ok=True)
|
||||
content = values if isinstance(values, str) else yaml.dump(values)
|
||||
with open(config_path, "w") as config_file:
|
||||
config_file.write(content)
|
||||
return config_path
|
||||
|
||||
|
||||
def test_plugin_paths_and_prefixes_are_namespaced(config_folder):
|
||||
assert plugin_config_folder() == str(config_folder / "plugins")
|
||||
assert plugin_config_path("example-plugin") == str(
|
||||
config_folder / "plugins" / "example-plugin.yaml"
|
||||
)
|
||||
assert plugin_env_prefix("example-plugin") == "MVT_PLUGIN_EXAMPLE_PLUGIN_"
|
||||
assert plugin_env_prefix("other-plugin") == "MVT_PLUGIN_OTHER_PLUGIN_"
|
||||
|
||||
|
||||
def test_defaults_are_used_without_file_or_environment(config_folder):
|
||||
settings = ExamplePluginSettings.load()
|
||||
|
||||
assert settings.API_KEY is None
|
||||
assert settings.CACHE_FOLDER == "cache"
|
||||
assert settings.MAX_RESULTS == 25
|
||||
assert not os.path.exists(plugin_config_path("example-plugin"))
|
||||
|
||||
|
||||
def test_values_are_loaded_from_the_plugin_file(config_folder):
|
||||
_write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5})
|
||||
|
||||
settings = ExamplePluginSettings.load()
|
||||
|
||||
assert settings.API_KEY == "from-file"
|
||||
assert settings.MAX_RESULTS == 5
|
||||
assert settings.CACHE_FOLDER == "cache"
|
||||
|
||||
|
||||
def test_environment_overrides_the_plugin_file(config_folder, monkeypatch):
|
||||
_write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5})
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment")
|
||||
|
||||
settings = ExamplePluginSettings.load()
|
||||
|
||||
assert settings.API_KEY == "from-environment"
|
||||
assert settings.MAX_RESULTS == 5
|
||||
|
||||
|
||||
def test_arguments_override_the_environment_and_the_plugin_file(
|
||||
config_folder, monkeypatch
|
||||
):
|
||||
_write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5})
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment")
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "10")
|
||||
|
||||
settings = ExamplePluginSettings(API_KEY="from-argument")
|
||||
|
||||
assert settings.API_KEY == "from-argument"
|
||||
assert settings.MAX_RESULTS == 10
|
||||
|
||||
|
||||
def test_save_and_load_round_trip(config_folder):
|
||||
settings = ExamplePluginSettings.load()
|
||||
settings.API_KEY = "saved-key"
|
||||
settings.MAX_RESULTS = 100
|
||||
|
||||
settings.save()
|
||||
|
||||
config_path = plugin_config_path("example-plugin")
|
||||
assert os.path.isfile(config_path)
|
||||
with open(config_path) as config_file:
|
||||
assert yaml.safe_load(config_file) == {
|
||||
"API_KEY": "saved-key",
|
||||
"MAX_RESULTS": 100,
|
||||
}
|
||||
|
||||
reloaded = ExamplePluginSettings.load()
|
||||
assert reloaded.API_KEY == "saved-key"
|
||||
assert reloaded.MAX_RESULTS == 100
|
||||
assert reloaded.CACHE_FOLDER == "cache"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="POSIX file permissions are not available"
|
||||
)
|
||||
def test_saved_file_is_only_readable_by_the_user(config_folder):
|
||||
settings = ExamplePluginSettings.load()
|
||||
settings.API_KEY = "saved-key"
|
||||
|
||||
settings.save()
|
||||
|
||||
config_path = plugin_config_path("example-plugin")
|
||||
assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600
|
||||
folder_mode = stat.S_IMODE(os.stat(plugin_config_folder()).st_mode)
|
||||
assert folder_mode & 0o077 == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="POSIX file permissions are not available"
|
||||
)
|
||||
def test_save_restricts_the_permissions_of_an_existing_file(config_folder):
|
||||
config_path = _write_plugin_file("example-plugin", {"API_KEY": "from-file"})
|
||||
os.chmod(config_path, 0o644)
|
||||
|
||||
settings = ExamplePluginSettings.load()
|
||||
settings.MAX_RESULTS = 100
|
||||
settings.save()
|
||||
|
||||
assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600
|
||||
assert os.listdir(plugin_config_folder()) == ["example-plugin.yaml"]
|
||||
|
||||
|
||||
def test_save_only_persists_non_default_values(config_folder):
|
||||
settings = ExamplePluginSettings.load()
|
||||
settings.CACHE_FOLDER = "another-cache"
|
||||
|
||||
settings.save()
|
||||
|
||||
with open(plugin_config_path("example-plugin")) as config_file:
|
||||
assert yaml.safe_load(config_file) == {"CACHE_FOLDER": "another-cache"}
|
||||
|
||||
|
||||
def test_save_does_not_persist_values_coming_from_the_environment(
|
||||
config_folder, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret")
|
||||
|
||||
settings = ExamplePluginSettings.load()
|
||||
assert settings.API_KEY == "environment-secret"
|
||||
settings.MAX_RESULTS = 100
|
||||
settings.save()
|
||||
|
||||
with open(plugin_config_path("example-plugin")) as config_file:
|
||||
assert yaml.safe_load(config_file) == {"MAX_RESULTS": 100}
|
||||
|
||||
|
||||
def test_an_invalid_environment_variable_still_protects_the_other_values(
|
||||
config_folder, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret")
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "not-an-int")
|
||||
|
||||
settings = ExamplePluginSettings(MAX_RESULTS=100)
|
||||
assert settings.API_KEY == "environment-secret"
|
||||
settings.save()
|
||||
|
||||
with open(plugin_config_path("example-plugin")) as config_file:
|
||||
saved_values = yaml.safe_load(config_file)
|
||||
assert saved_values == {"MAX_RESULTS": 100}
|
||||
assert "API_KEY" not in saved_values
|
||||
|
||||
|
||||
def test_save_persists_values_which_differ_from_the_environment(
|
||||
config_folder, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret")
|
||||
|
||||
settings = ExamplePluginSettings.load()
|
||||
settings.API_KEY = "chosen-key"
|
||||
settings.save()
|
||||
|
||||
with open(plugin_config_path("example-plugin")) as config_file:
|
||||
assert yaml.safe_load(config_file) == {"API_KEY": "chosen-key"}
|
||||
|
||||
|
||||
def test_save_does_not_write_the_mvt_configuration_file(config_folder):
|
||||
settings = ExamplePluginSettings.load()
|
||||
settings.API_KEY = "saved-key"
|
||||
|
||||
settings.save()
|
||||
|
||||
assert os.listdir(config_folder) == ["plugins"]
|
||||
|
||||
|
||||
def test_unknown_keys_in_the_plugin_file_are_ignored(config_folder):
|
||||
_write_plugin_file(
|
||||
"example-plugin",
|
||||
{"API_KEY": "from-file", "UNKNOWN_SETTING": "ignored"},
|
||||
)
|
||||
|
||||
settings = ExamplePluginSettings.load()
|
||||
|
||||
assert settings.API_KEY == "from-file"
|
||||
assert not hasattr(settings, "UNKNOWN_SETTING")
|
||||
|
||||
|
||||
def test_unparsable_plugin_file_is_reported_with_its_path(config_folder):
|
||||
config_path = _write_plugin_file("example-plugin", "API_KEY: [unclosed\n")
|
||||
|
||||
with pytest.raises(PluginConfigLoadError) as raised:
|
||||
ExamplePluginSettings.load()
|
||||
|
||||
assert config_path in str(raised.value)
|
||||
|
||||
|
||||
def test_plugin_file_which_is_not_a_mapping_is_reported_with_its_path(config_folder):
|
||||
config_path = _write_plugin_file("example-plugin", "- one\n- two\n")
|
||||
|
||||
with pytest.raises(PluginConfigLoadError) as raised:
|
||||
ExamplePluginSettings.load()
|
||||
|
||||
assert config_path in str(raised.value)
|
||||
assert "mapping of setting names" in str(raised.value)
|
||||
|
||||
|
||||
def test_plugins_do_not_interfere_with_each_other(config_folder, monkeypatch):
|
||||
_write_plugin_file("other-plugin", {"API_KEY": "other-file-key"})
|
||||
monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "example-environment-key")
|
||||
|
||||
example_settings = ExamplePluginSettings.load()
|
||||
other_settings = OtherPluginSettings.load()
|
||||
|
||||
assert example_settings.API_KEY == "example-environment-key"
|
||||
assert other_settings.API_KEY == "other-file-key"
|
||||
|
||||
example_settings.MAX_RESULTS = 100
|
||||
example_settings.save()
|
||||
assert sorted(os.listdir(plugin_config_folder())) == [
|
||||
"example-plugin.yaml",
|
||||
"other-plugin.yaml",
|
||||
]
|
||||
with open(plugin_config_path("other-plugin")) as config_file:
|
||||
assert yaml.safe_load(config_file) == {"API_KEY": "other-file-key"}
|
||||
|
||||
|
||||
def test_subclass_without_plugin_name_is_rejected():
|
||||
with pytest.raises(TypeError, match="plugin_name"):
|
||||
|
||||
class MissingNameSettings(MVTPluginSettings):
|
||||
API_KEY: Optional[str] = None
|
||||
|
||||
|
||||
def test_subclass_with_invalid_plugin_name_is_rejected():
|
||||
with pytest.raises(ValueError, match="Invalid plugin name"):
|
||||
|
||||
class InvalidNameSettings(MVTPluginSettings):
|
||||
plugin_name = "Bad/Name"
|
||||
|
||||
|
||||
def test_underscores_are_not_allowed_in_plugin_names():
|
||||
# Underscores are replaced by dashes in the environment prefix, so allowing
|
||||
# both would let two plugin names share one environment namespace.
|
||||
with pytest.raises(ValueError, match="Invalid plugin name"):
|
||||
|
||||
class UnderscoreNameSettings(MVTPluginSettings):
|
||||
plugin_name = "under_score"
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid plugin name"):
|
||||
plugin_config_path("under_score")
|
||||
with pytest.raises(ValueError, match="Invalid plugin name"):
|
||||
plugin_env_prefix("under_score")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""]
|
||||
)
|
||||
def test_unsafe_plugin_names_have_no_configuration_path(plugin_name):
|
||||
with pytest.raises(ValueError, match="Invalid plugin name"):
|
||||
plugin_config_path(plugin_name)
|
||||
|
||||
|
||||
def test_data_folder_is_namespaced_and_created(data_folder):
|
||||
folder = plugin_data_folder("example-plugin")
|
||||
|
||||
assert folder == str(data_folder / "plugin-data" / "example-plugin")
|
||||
assert os.path.isdir(folder)
|
||||
|
||||
|
||||
def test_data_folder_can_be_requested_repeatedly(data_folder):
|
||||
folder = plugin_data_folder("example-plugin")
|
||||
with open(os.path.join(folder, "kept.json"), "w") as data_file:
|
||||
data_file.write("{}")
|
||||
|
||||
assert plugin_data_folder("example-plugin") == folder
|
||||
assert os.listdir(folder) == ["kept.json"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="POSIX file permissions are not available"
|
||||
)
|
||||
def test_data_folder_is_only_accessible_by_the_user(data_folder):
|
||||
folder = plugin_data_folder("example-plugin")
|
||||
|
||||
assert stat.S_IMODE(os.stat(folder).st_mode) & 0o077 == 0
|
||||
parent_mode = stat.S_IMODE(os.stat(os.path.dirname(folder)).st_mode)
|
||||
assert parent_mode & 0o077 == 0
|
||||
|
||||
|
||||
def test_plugins_get_their_own_data_folder(data_folder):
|
||||
example_folder = plugin_data_folder("example-plugin")
|
||||
other_folder = plugin_data_folder("other-plugin")
|
||||
|
||||
assert example_folder != other_folder
|
||||
assert sorted(os.listdir(data_folder / "plugin-data")) == [
|
||||
"example-plugin",
|
||||
"other-plugin",
|
||||
]
|
||||
|
||||
|
||||
def test_data_folder_does_not_touch_the_configuration_folder(
|
||||
config_folder, data_folder
|
||||
):
|
||||
plugin_data_folder("example-plugin")
|
||||
|
||||
assert not os.path.exists(plugin_config_folder())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""]
|
||||
)
|
||||
def test_unsafe_plugin_names_have_no_data_folder(data_folder, plugin_name):
|
||||
with pytest.raises(ValueError, match="Invalid plugin name"):
|
||||
plugin_data_folder(plugin_name)
|
||||
|
||||
assert not os.path.exists(data_folder)
|
||||
|
||||
|
||||
def test_settings_class_knows_its_data_folder(data_folder):
|
||||
folder = ExamplePluginSettings.data_folder()
|
||||
|
||||
assert folder == plugin_data_folder("example-plugin")
|
||||
assert os.path.isdir(folder)
|
||||
assert OtherPluginSettings.data_folder() != folder
|
||||
|
||||
|
||||
def test_settings_instance_uses_the_same_data_folder(config_folder, data_folder):
|
||||
settings = ExamplePluginSettings.load()
|
||||
|
||||
assert settings.data_folder() == ExamplePluginSettings.data_folder()
|
||||
|
||||
|
||||
def test_subclass_without_its_own_name_shares_the_data_folder(data_folder):
|
||||
class InheritingSettings(ExamplePluginSettings):
|
||||
pass
|
||||
|
||||
assert InheritingSettings.data_folder() == ExamplePluginSettings.data_folder()
|
||||
@@ -0,0 +1,711 @@
|
||||
import json
|
||||
import shlex
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from mvt.common import logo
|
||||
from mvt.common.cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP
|
||||
from mvt.common.updates import (
|
||||
MVTUpdates,
|
||||
PluginUpdates,
|
||||
installed_plugin_distributions,
|
||||
)
|
||||
|
||||
|
||||
REPOSITORY_URL = "https://example.org/plugin.git"
|
||||
INSTALLED_COMMIT = "a" * 40
|
||||
REMOTE_COMMIT = "b" * 40
|
||||
|
||||
|
||||
class FakeDistribution:
|
||||
def __init__(self, name, version="1.0.0", direct_url=None):
|
||||
self.name = name
|
||||
self.version = version
|
||||
self.direct_url = direct_url
|
||||
|
||||
def read_text(self, file_name):
|
||||
if file_name == "direct_url.json" and self.direct_url is not None:
|
||||
return json.dumps(self.direct_url)
|
||||
return None
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, status_code=200, payload=None):
|
||||
self.status_code = status_code
|
||||
self.payload = payload or {}
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
def _entry_point(name, distribution, value="plugin:modules"):
|
||||
return SimpleNamespace(name=name, value=value, dist=distribution)
|
||||
|
||||
|
||||
def _git_distribution(requested_revision=None, commit=INSTALLED_COMMIT):
|
||||
vcs_info = {"vcs": "git", "commit_id": commit}
|
||||
if requested_revision:
|
||||
vcs_info["requested_revision"] = requested_revision
|
||||
|
||||
return FakeDistribution(
|
||||
"example-plugin",
|
||||
direct_url={"url": REPOSITORY_URL, "vcs_info": vcs_info},
|
||||
)
|
||||
|
||||
|
||||
def _fake_git(stdout="", returncode=0, exception=None, calls=None):
|
||||
def run(command, **kwargs):
|
||||
if calls is not None:
|
||||
calls.append((command, kwargs))
|
||||
if exception is not None:
|
||||
raise exception
|
||||
return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="")
|
||||
|
||||
return run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def data_folder(tmp_path, monkeypatch):
|
||||
folder = tmp_path / "mvt-data"
|
||||
monkeypatch.setattr("mvt.common.updates.MVT_DATA_FOLDER", str(folder))
|
||||
return folder
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def one_plugin(monkeypatch):
|
||||
def install(distribution):
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.installed_plugin_distributions",
|
||||
lambda: [distribution],
|
||||
)
|
||||
return distribution
|
||||
|
||||
return install
|
||||
|
||||
|
||||
def test_installed_plugin_distributions_covers_every_plugin_group(monkeypatch):
|
||||
zeta = FakeDistribution("zeta-plugin")
|
||||
alpha = FakeDistribution("alpha-plugin")
|
||||
neutral = FakeDistribution("neutral-plugin")
|
||||
|
||||
def entry_points(*, group):
|
||||
if group == MODULES_ENTRY_POINT_GROUP:
|
||||
return [_entry_point("zeta", zeta), _entry_point("alpha", alpha)]
|
||||
if group == IOS_CLI_PLUGIN_GROUP:
|
||||
return [_entry_point("zeta-ios", zeta)]
|
||||
if group == ANDROID_CLI_PLUGIN_GROUP:
|
||||
return [_entry_point("alpha-android", alpha)]
|
||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
||||
return [_entry_point("shared", neutral)]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.importlib.metadata.entry_points", entry_points
|
||||
)
|
||||
|
||||
distributions = installed_plugin_distributions()
|
||||
|
||||
assert [distribution.name for distribution in distributions] == [
|
||||
"alpha-plugin",
|
||||
"neutral-plugin",
|
||||
"zeta-plugin",
|
||||
]
|
||||
|
||||
|
||||
def test_installed_plugin_distributions_skips_mvt_and_orphan_entry_points(monkeypatch):
|
||||
entry_points = [
|
||||
_entry_point("builtin", FakeDistribution("mvt")),
|
||||
SimpleNamespace(name="orphan", value="plugin:modules", dist=None),
|
||||
_entry_point("plugin", FakeDistribution("example-plugin")),
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.importlib.metadata.entry_points",
|
||||
lambda **kwargs: entry_points,
|
||||
)
|
||||
|
||||
distributions = installed_plugin_distributions()
|
||||
|
||||
assert [distribution.name for distribution in distributions] == ["example-plugin"]
|
||||
|
||||
|
||||
def test_installed_plugin_distributions_survives_broken_metadata(monkeypatch, caplog):
|
||||
def entry_points(*, group):
|
||||
raise RuntimeError("invalid package metadata")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.importlib.metadata.entry_points", entry_points
|
||||
)
|
||||
|
||||
assert installed_plugin_distributions() == []
|
||||
assert "Unable to discover installed plugin packages" in caplog.text
|
||||
|
||||
|
||||
def test_index_plugin_update_is_reported(monkeypatch, data_folder, one_plugin):
|
||||
one_plugin(FakeDistribution("example-plugin", version="1.0.0"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.requests.get",
|
||||
lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}),
|
||||
)
|
||||
|
||||
findings = PluginUpdates().check()
|
||||
|
||||
assert findings == [
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_index_plugin_queries_the_package_index_with_the_configured_timeout(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(FakeDistribution("example-plugin"))
|
||||
requests_made = []
|
||||
|
||||
def get(url, **kwargs):
|
||||
requests_made.append((url, kwargs))
|
||||
return FakeResponse(payload={"info": {"version": "1.0.0"}})
|
||||
|
||||
monkeypatch.setattr("mvt.common.updates.requests.get", get)
|
||||
monkeypatch.setattr("mvt.common.updates.settings.NETWORK_TIMEOUT", 3)
|
||||
|
||||
PluginUpdates().check()
|
||||
|
||||
assert requests_made == [
|
||||
("https://pypi.org/pypi/example-plugin/json", {"timeout": 3})
|
||||
]
|
||||
|
||||
|
||||
def test_up_to_date_index_plugin_is_not_reported(monkeypatch, data_folder, one_plugin):
|
||||
one_plugin(FakeDistribution("example-plugin", version="1.2.0"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.requests.get",
|
||||
lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}),
|
||||
)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_unpublished_plugin_is_skipped_silently(monkeypatch, data_folder, one_plugin):
|
||||
one_plugin(FakeDistribution("private-plugin"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.requests.get",
|
||||
lambda url, **kwargs: FakeResponse(status_code=404),
|
||||
)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_repository_plugin_following_a_branch_is_reported(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision="main"))
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls),
|
||||
)
|
||||
monkeypatch.delenv("GIT_SSH_COMMAND", raising=False)
|
||||
|
||||
findings = PluginUpdates().check()
|
||||
|
||||
assert findings == [
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "aaaaaaaa",
|
||||
"latest": "bbbbbbbb",
|
||||
"origin": "git",
|
||||
"upgrade_command": (
|
||||
f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@main'"
|
||||
),
|
||||
}
|
||||
]
|
||||
command, options = calls[0]
|
||||
assert command == ["git", "ls-remote", REPOSITORY_URL, "main"]
|
||||
assert options["env"]["GIT_TERMINAL_PROMPT"] == "0"
|
||||
# ssh asks the terminal for a passphrase or a host key unless it is told
|
||||
# not to, which git itself cannot prevent.
|
||||
assert options["env"]["GIT_SSH_COMMAND"] == (
|
||||
"ssh -o BatchMode=yes -o ConnectTimeout=10"
|
||||
)
|
||||
|
||||
|
||||
def test_batch_mode_options_come_before_the_configured_ssh_options(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision="main"))
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls),
|
||||
)
|
||||
monkeypatch.setenv("GIT_SSH_COMMAND", "ssh -o BatchMode=no -i /home/analyst/key")
|
||||
|
||||
PluginUpdates().check()
|
||||
|
||||
# ssh uses the first value it is given for a keyword, so an analyst asking
|
||||
# for prompts cannot bring them back, while their other options still
|
||||
# apply.
|
||||
assert calls[0][1]["env"]["GIT_SSH_COMMAND"] == (
|
||||
"ssh -o BatchMode=yes -o ConnectTimeout=10 -o BatchMode=no -i /home/analyst/key"
|
||||
)
|
||||
|
||||
|
||||
def test_repository_plugin_without_a_revision_follows_the_default_branch(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution())
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout=f"{REMOTE_COMMIT}\tHEAD\n", calls=calls),
|
||||
)
|
||||
|
||||
findings = PluginUpdates().check()
|
||||
|
||||
assert calls[0][0] == ["git", "ls-remote", REPOSITORY_URL, "HEAD"]
|
||||
assert findings[0]["upgrade_command"] == (
|
||||
f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}'"
|
||||
)
|
||||
|
||||
|
||||
def test_hostile_revision_cannot_inject_into_the_upgrade_command(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
revision = "main$(id)`id`;id"
|
||||
one_plugin(_git_distribution(requested_revision=revision))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/{revision}\n"),
|
||||
)
|
||||
|
||||
upgrade_command = PluginUpdates().check()[0]["upgrade_command"]
|
||||
|
||||
# Single quotes are the only quoting a shell does not expand anything in.
|
||||
assert upgrade_command == (
|
||||
f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@{revision}'"
|
||||
)
|
||||
assert shlex.split(upgrade_command) == [
|
||||
"pip",
|
||||
"install",
|
||||
"-U",
|
||||
f"example-plugin @ git+{REPOSITORY_URL}@{revision}",
|
||||
]
|
||||
|
||||
|
||||
def test_repository_plugin_with_an_option_like_url_is_skipped(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(
|
||||
FakeDistribution(
|
||||
"example-plugin",
|
||||
direct_url={
|
||||
"url": "--upload-pack=touch /tmp/mvt",
|
||||
"vcs_info": {"vcs": "git", "commit_id": INSTALLED_COMMIT},
|
||||
},
|
||||
)
|
||||
)
|
||||
calls = []
|
||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_repository_plugin_with_an_option_like_revision_is_skipped(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision="--upload-pack=touch /tmp/mvt"))
|
||||
calls = []
|
||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_repository_plugin_at_the_latest_commit_is_not_reported(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision="main"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout=f"{INSTALLED_COMMIT}\trefs/heads/main\n"),
|
||||
)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_commit_pinned_repository_plugin_is_never_outdated(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT))
|
||||
calls = []
|
||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_short_commit_pinned_repository_plugin_is_never_outdated(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT[:10]))
|
||||
calls = []
|
||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_tag_pinned_repository_plugin_is_never_outdated(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision="v1.0.0"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/tags/v1.0.0\n"),
|
||||
)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_repository_plugin_is_skipped_without_git(monkeypatch, data_folder, one_plugin):
|
||||
one_plugin(_git_distribution(requested_revision="main"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(exception=FileNotFoundError("git")),
|
||||
)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_repository_plugin_is_skipped_when_git_fails(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(_git_distribution(requested_revision="main"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.subprocess.run",
|
||||
_fake_git(stdout="", returncode=128),
|
||||
)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_local_plugin_install_is_skipped(monkeypatch, data_folder, one_plugin):
|
||||
one_plugin(
|
||||
FakeDistribution(
|
||||
"example-plugin",
|
||||
direct_url={
|
||||
"url": "file:///home/analyst/example-plugin",
|
||||
"dir_info": {"editable": True},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise AssertionError("a local plugin install must not be checked")
|
||||
|
||||
monkeypatch.setattr("mvt.common.updates.requests.get", fail)
|
||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", fail)
|
||||
|
||||
assert PluginUpdates().check() == []
|
||||
|
||||
|
||||
def test_check_stores_the_findings_and_the_check_timestamp(
|
||||
monkeypatch, data_folder, one_plugin
|
||||
):
|
||||
one_plugin(FakeDistribution("example-plugin", version="1.0.0"))
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.requests.get",
|
||||
lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}),
|
||||
)
|
||||
plugin_updates = PluginUpdates()
|
||||
|
||||
findings = plugin_updates.check()
|
||||
|
||||
assert json.loads((data_folder / "plugin_updates.json").read_text()) == findings
|
||||
assert (data_folder / "latest_plugins_check").read_text().isdigit()
|
||||
assert PluginUpdates().get_findings() == findings
|
||||
|
||||
|
||||
def test_findings_are_empty_before_the_first_check(data_folder):
|
||||
assert PluginUpdates().get_findings() == []
|
||||
|
||||
|
||||
def test_malformed_cached_findings_are_dropped(data_folder):
|
||||
plugin_updates = PluginUpdates()
|
||||
usable = {
|
||||
"name": "example-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
}
|
||||
data_folder.mkdir(parents=True, exist_ok=True)
|
||||
(data_folder / "plugin_updates.json").write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{"oops": 1},
|
||||
"not a finding",
|
||||
{"name": "half-plugin", "installed": "1.0.0"},
|
||||
{**usable, "latest": None},
|
||||
usable,
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert plugin_updates.get_findings() == [usable]
|
||||
|
||||
|
||||
def test_corrupt_cached_findings_are_ignored(data_folder):
|
||||
data_folder.mkdir(parents=True, exist_ok=True)
|
||||
(data_folder / "plugin_updates.json").write_text("{ not json", encoding="utf-8")
|
||||
|
||||
assert PluginUpdates().get_findings() == []
|
||||
|
||||
|
||||
def test_cached_findings_of_upgraded_and_removed_plugins_are_dropped(
|
||||
monkeypatch, data_folder
|
||||
):
|
||||
findings = [
|
||||
{
|
||||
"name": "upgraded-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U upgraded-plugin",
|
||||
},
|
||||
{
|
||||
"name": "removed-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U removed-plugin",
|
||||
},
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
},
|
||||
]
|
||||
plugin_updates = PluginUpdates()
|
||||
plugin_updates.set_findings(findings)
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.updates.installed_plugin_distributions",
|
||||
lambda: [
|
||||
# The analyst upgraded this plugin since the latest check.
|
||||
FakeDistribution("upgraded-plugin", version="1.2.0"),
|
||||
FakeDistribution("example-plugin", version="1.0.0"),
|
||||
],
|
||||
)
|
||||
|
||||
assert plugin_updates.current_findings() == [findings[2]]
|
||||
|
||||
|
||||
def test_cached_findings_of_updated_repository_plugins_are_dropped(data_folder):
|
||||
findings = [
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "aaaaaaaa",
|
||||
"latest": "bbbbbbbb",
|
||||
"origin": "git",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
}
|
||||
]
|
||||
plugin_updates = PluginUpdates()
|
||||
plugin_updates.set_findings(findings)
|
||||
|
||||
assert plugin_updates.current_findings([_git_distribution()]) == findings
|
||||
assert (
|
||||
plugin_updates.current_findings([_git_distribution(commit=REMOTE_COMMIT)]) == []
|
||||
)
|
||||
|
||||
|
||||
def test_corrupt_check_timestamp_does_not_raise(data_folder):
|
||||
plugin_updates = PluginUpdates()
|
||||
data_folder.mkdir(parents=True, exist_ok=True)
|
||||
(data_folder / "latest_plugins_check").write_text("truncated", encoding="utf-8")
|
||||
|
||||
assert plugin_updates.get_latest_check() == 0
|
||||
assert plugin_updates.should_check() == (True, 0)
|
||||
|
||||
|
||||
def test_should_check_is_throttled_for_twelve_hours(data_folder):
|
||||
plugin_updates = PluginUpdates()
|
||||
plugin_updates.set_findings([])
|
||||
|
||||
recent = datetime.now() - timedelta(hours=4)
|
||||
with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(str(int(recent.timestamp())))
|
||||
|
||||
should_check, hours = plugin_updates.should_check()
|
||||
assert not should_check
|
||||
assert hours == 8
|
||||
|
||||
old = datetime.now() - timedelta(hours=13)
|
||||
with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(str(int(old.timestamp())))
|
||||
|
||||
assert plugin_updates.should_check() == (True, 0)
|
||||
|
||||
|
||||
def test_should_check_without_a_previous_check(data_folder):
|
||||
assert PluginUpdates().should_check() == (True, 0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_version_check(monkeypatch):
|
||||
monkeypatch.setattr(MVTUpdates, "check", lambda self: "")
|
||||
# Keep rich from wrapping the plugin lines while they are being asserted.
|
||||
monkeypatch.setenv("COLUMNS", "200")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def throttled_cache(monkeypatch, data_folder):
|
||||
"""Fill the findings cache and put the check inside its throttle window."""
|
||||
|
||||
def fill(findings, distributions):
|
||||
PluginUpdates().set_findings(findings)
|
||||
monkeypatch.setattr(
|
||||
logo, "installed_plugin_distributions", lambda: distributions
|
||||
)
|
||||
monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8))
|
||||
monkeypatch.setattr(
|
||||
PluginUpdates,
|
||||
"check",
|
||||
lambda self: pytest.fail("the check must be throttled"),
|
||||
)
|
||||
|
||||
return fill
|
||||
|
||||
|
||||
def test_logo_prints_the_cached_plugin_updates(
|
||||
capsys, no_version_check, throttled_cache
|
||||
):
|
||||
throttled_cache(
|
||||
[
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
}
|
||||
],
|
||||
[FakeDistribution("example-plugin", version="1.0.0")],
|
||||
)
|
||||
|
||||
logo.check_updates(disable_indicator_check=True)
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert "Plugin updates available:" in output
|
||||
assert "example-plugin 1.0.0 → 1.2.0 (pip install -U example-plugin)" in output
|
||||
|
||||
|
||||
def test_logo_does_not_print_a_cached_update_of_an_upgraded_plugin(
|
||||
capsys, no_version_check, throttled_cache
|
||||
):
|
||||
throttled_cache(
|
||||
[
|
||||
{
|
||||
"name": "example-plugin",
|
||||
"installed": "1.0.0",
|
||||
"latest": "1.2.0",
|
||||
"origin": "pypi",
|
||||
"upgrade_command": "pip install -U example-plugin",
|
||||
}
|
||||
],
|
||||
# The analyst already upgraded the plugin the cached finding is about.
|
||||
[FakeDistribution("example-plugin", version="1.2.0")],
|
||||
)
|
||||
|
||||
logo.check_updates(disable_indicator_check=True)
|
||||
|
||||
assert "Plugin updates" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_logo_prints_nothing_when_throttled_without_findings(
|
||||
capsys, no_version_check, throttled_cache
|
||||
):
|
||||
throttled_cache([], [FakeDistribution("example-plugin")])
|
||||
|
||||
logo.check_updates(disable_indicator_check=True)
|
||||
|
||||
assert "Plugin updates" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_logo_survives_a_corrupt_plugin_cache(
|
||||
monkeypatch, capsys, data_folder, no_version_check
|
||||
):
|
||||
data_folder.mkdir(parents=True, exist_ok=True)
|
||||
(data_folder / "plugin_updates.json").write_text(
|
||||
json.dumps([{"oops": 1}]), encoding="utf-8"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
logo,
|
||||
"installed_plugin_distributions",
|
||||
lambda: [FakeDistribution("example-plugin")],
|
||||
)
|
||||
monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8))
|
||||
|
||||
logo.check_updates(disable_indicator_check=True)
|
||||
|
||||
assert "Plugin updates" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_logo_skips_the_plugin_check_without_plugins(
|
||||
monkeypatch, capsys, no_version_check
|
||||
):
|
||||
monkeypatch.setattr(logo, "installed_plugin_distributions", list)
|
||||
monkeypatch.setattr(
|
||||
PluginUpdates,
|
||||
"should_check",
|
||||
lambda self: pytest.fail("plugins must not be checked without plugins"),
|
||||
)
|
||||
|
||||
logo.check_updates(disable_indicator_check=True)
|
||||
|
||||
assert "Plugin updates" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_logo_skips_the_plugin_check_without_network_access(
|
||||
monkeypatch, capsys, no_version_check
|
||||
):
|
||||
monkeypatch.setattr("mvt.common.logo.settings.NETWORK_ACCESS_ALLOWED", False)
|
||||
monkeypatch.setattr(
|
||||
logo,
|
||||
"installed_plugin_distributions",
|
||||
lambda: pytest.fail("plugins must not be listed without network access"),
|
||||
)
|
||||
|
||||
logo.check_updates(disable_indicator_check=True)
|
||||
|
||||
assert "Plugin updates" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_logo_skips_the_plugin_check_when_update_checks_are_disabled(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
logo,
|
||||
"installed_plugin_distributions",
|
||||
lambda: pytest.fail("plugins must not be checked with --disable-update-check"),
|
||||
)
|
||||
|
||||
logo.check_updates(disable_version_check=True, disable_indicator_check=True)
|
||||
|
||||
assert capsys.readouterr().out == ""
|
||||
@@ -8,6 +8,7 @@ import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from mvt.common.log import MVTLogHandler
|
||||
from mvt.common.utils import (
|
||||
CustomJSONEncoder,
|
||||
convert_datetime_to_iso,
|
||||
@@ -16,6 +17,7 @@ from mvt.common.utils import (
|
||||
convert_unix_to_utc_datetime,
|
||||
generate_hashes_from_path,
|
||||
get_sha256_from_file_path,
|
||||
init_logging,
|
||||
)
|
||||
|
||||
from ..utils import get_artifact_folder
|
||||
@@ -103,3 +105,20 @@ class TestCustomJSONEncoder:
|
||||
json.dumps({"name": "家".encode()}, cls=CustomJSONEncoder)
|
||||
== '{"name": "\\u5bb6"}'
|
||||
)
|
||||
|
||||
|
||||
class TestInitLogging:
|
||||
def test__init_logging_is_idempotent(self):
|
||||
# Loaded module packages may import an MVT CLI module, which calls
|
||||
# init_logging() again at import time. A second call must not add
|
||||
# a duplicate console handler.
|
||||
log = logging.getLogger("mvt")
|
||||
init_logging()
|
||||
handler_count = sum(
|
||||
isinstance(handler, MVTLogHandler) for handler in log.handlers
|
||||
)
|
||||
init_logging()
|
||||
assert (
|
||||
sum(isinstance(handler, MVTLogHandler) for handler in log.handlers)
|
||||
== handler_count
|
||||
)
|
||||
@@ -8,6 +8,11 @@ import os
|
||||
|
||||
import pytest
|
||||
|
||||
from mvt.common.cli_plugins import (
|
||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||
MVT_CUSTOM_COMMANDS_ENV,
|
||||
MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||
)
|
||||
from mvt.common.indicators import Indicators
|
||||
|
||||
from .artifacts.generate_stix import generate_test_stix_file
|
||||
@@ -58,3 +63,33 @@ def indicators_factory(indicator_file):
|
||||
return ind
|
||||
|
||||
return f
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def restore_cli_commands(monkeypatch):
|
||||
"""Keep the external commands a test registers out of the next test.
|
||||
|
||||
Each CLI group is a module-level object shared by every test, so a test
|
||||
registering plugin or environment commands on one has to put it back. The
|
||||
groups are imported here rather than at the top of the file, so that
|
||||
collecting the tests does not import three CLIs for the sake of one
|
||||
fixture.
|
||||
"""
|
||||
from mvt.android.cli import cli as android_cli
|
||||
from mvt.cli import cli as neutral_cli
|
||||
from mvt.ios.cli import cli as ios_cli
|
||||
|
||||
groups = (neutral_cli, ios_cli, android_cli)
|
||||
for variable in (
|
||||
MVT_CUSTOM_COMMANDS_ENV,
|
||||
MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||
):
|
||||
monkeypatch.delenv(variable, raising=False)
|
||||
originals = [dict(group.commands) for group in groups]
|
||||
yield
|
||||
for group, commands in zip(groups, originals):
|
||||
group.commands.clear()
|
||||
group.commands.update(commands)
|
||||
if hasattr(group, "_mvt_external_command_sources"):
|
||||
delattr(group, "_mvt_external_command_sources")
|
||||
@@ -0,0 +1,105 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
from mvt.common.module import run_module
|
||||
from mvt.ios.modules.mixed.interactionc import InteractionC
|
||||
from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts
|
||||
|
||||
from ..utils import get_ios_backup_folder
|
||||
|
||||
|
||||
class TestInteractionCModule:
|
||||
def test_extraction_with_whatsapp_contacts(self):
|
||||
contacts = WhatsappContacts(target_path=get_ios_backup_folder())
|
||||
run_module(contacts)
|
||||
|
||||
m = InteractionC(target_path=get_ios_backup_folder())
|
||||
m.dependency_modules = {WhatsappContacts: contacts}
|
||||
run_module(m)
|
||||
|
||||
assert len(m.results) == 3
|
||||
|
||||
incoming = next(
|
||||
r for r in m.results if r["sender_identifier"] == "100000000000001@lid"
|
||||
)
|
||||
assert incoming["direction"] == "INCOMING"
|
||||
assert incoming["sender_resolved_phone_number"] == "+14155550100"
|
||||
assert incoming["sender_resolved_name"] == "Alice Example"
|
||||
|
||||
outgoing = next(
|
||||
r for r in m.results if r["direction"] == "OUTGOING"
|
||||
)
|
||||
assert outgoing["recipient_identifier"] == "+14155550100"
|
||||
assert outgoing["recipient_resolved_name"] == "Alice Example"
|
||||
assert outgoing["domain_resolved_phone_number"] == "+14155550100"
|
||||
assert outgoing["domain_resolved_name"] == "Alice Example"
|
||||
|
||||
sms = next(
|
||||
r for r in m.results if r["bundle_id"] == "com.apple.MobileSMS"
|
||||
)
|
||||
assert sms.get("sender_resolved_name") is None
|
||||
assert sms["sender_display_name"] == "Bob Example"
|
||||
|
||||
events = [entry["data"] for entry in m.timeline]
|
||||
assert (
|
||||
"[net.whatsapp.WhatsApp] INCOMING from "
|
||||
"Alice Example (+14155550100) to local user" in events
|
||||
)
|
||||
assert (
|
||||
"[net.whatsapp.WhatsApp] OUTGOING from local user to "
|
||||
"Alice Example (+14155550100)" in events
|
||||
)
|
||||
assert (
|
||||
"[com.apple.MobileSMS] INCOMING from "
|
||||
"Bob Example (+14155550101) to local user" in events
|
||||
)
|
||||
|
||||
# The creation date is only serialized when it diverges from the
|
||||
# start date; the SMS record was created 90 days after the event.
|
||||
creation_events = [
|
||||
entry
|
||||
for entry in m.timeline
|
||||
if entry["event"] == "interactions_creation_date"
|
||||
]
|
||||
assert len(creation_events) == 1
|
||||
assert creation_events[0]["timestamp"] == "2025-12-09 12:26:40.000000"
|
||||
assert creation_events[0]["data"] == (
|
||||
"Interaction record created 90 days after the event: "
|
||||
"[com.apple.MobileSMS] INCOMING from "
|
||||
"Bob Example (+14155550101) to local user"
|
||||
)
|
||||
|
||||
# Per-contact aggregate dates use contact-centric data strings.
|
||||
first_seen = [
|
||||
entry
|
||||
for entry in m.timeline
|
||||
if entry["event"] == "first_incoming_sender_date"
|
||||
]
|
||||
assert len(first_seen) == 1
|
||||
assert first_seen[0]["timestamp"] == "2025-09-03 13:46:40.000000"
|
||||
assert first_seen[0]["data"] == (
|
||||
"First incoming interaction from Bob Example (+14155550101)"
|
||||
)
|
||||
assert (
|
||||
"Last incoming interaction from Bob Example (+14155550101)"
|
||||
in events
|
||||
)
|
||||
|
||||
def test_extraction_without_whatsapp_contacts(self):
|
||||
# Without the WhatsappContacts dependency the module still runs, and
|
||||
# unresolvable LIDs are shown as-is.
|
||||
m = InteractionC(target_path=get_ios_backup_folder())
|
||||
run_module(m)
|
||||
|
||||
assert len(m.results) == 3
|
||||
events = [entry["data"] for entry in m.timeline]
|
||||
assert (
|
||||
"[net.whatsapp.WhatsApp] INCOMING from "
|
||||
"100000000000001@lid to local user" in events
|
||||
)
|
||||
assert (
|
||||
"[net.whatsapp.WhatsApp] OUTGOING from local user to "
|
||||
"+14155550100" in events
|
||||
)
|
||||
@@ -6,8 +6,79 @@
|
||||
import logging
|
||||
|
||||
from mvt.common.indicators import Indicators
|
||||
from mvt.common.module import run_module
|
||||
from mvt.ios.modules.mixed.whatsapp import Whatsapp
|
||||
|
||||
from ..utils import get_ios_backup_folder
|
||||
|
||||
|
||||
def test_extraction():
|
||||
m = Whatsapp(target_path=get_ios_backup_folder())
|
||||
run_module(m)
|
||||
|
||||
messages = [r for r in m.results if "ZTEXT" in r]
|
||||
sessions = [r for r in m.results if r.get("record_type") == "chat_session"]
|
||||
pairs = [
|
||||
r for r in m.results
|
||||
if r.get("record_type") == "lid_phone_number_pair"
|
||||
]
|
||||
assert len(messages) == 3
|
||||
assert len(sessions) == 2
|
||||
assert len(pairs) == 1
|
||||
|
||||
assert pairs[0]["lid"] == "100000000000001"
|
||||
assert pairs[0]["phone_number"] == "14155550100"
|
||||
assert pairs[0]["pair_timestamp"] == "2025-08-25 07:33:20.000000"
|
||||
|
||||
linked = next(r for r in messages if r.get("links"))
|
||||
assert linked["links"] == ["https://example.org/news"]
|
||||
|
||||
alice = next(s for s in sessions if s["partner_name"] == "Alice Example")
|
||||
assert alice["contact_jid"] == "100000000000001@lid"
|
||||
assert alice["partner_resolved_phone_number"] == "+14155550100"
|
||||
assert alice["first_stored_message_date"] == "2025-08-27 15:06:40.000000"
|
||||
assert alice["last_message_date"] == "2025-08-28 18:53:20.000000"
|
||||
assert alice["group_creation_date"] is None
|
||||
assert alice["stored_message_count"] == 2
|
||||
|
||||
group = next(s for s in sessions if s["partner_name"] == "Example Group")
|
||||
assert group["group_creation_date"] == "2025-08-21 20:13:20.000000"
|
||||
assert group["first_stored_message_date"] == "2025-08-29 22:40:00.000000"
|
||||
# The last stored message predates the session's own last-message date:
|
||||
# the newest message in this chat was deleted.
|
||||
assert group["last_stored_message_date"] == "2025-08-29 22:40:00.000000"
|
||||
assert group["last_message_date"] == "2025-08-31 02:26:40.000000"
|
||||
|
||||
# 3 message events, first/last per chat, the group creation and the
|
||||
# LID-phone number pair.
|
||||
assert len(m.timeline) == 9
|
||||
events = {
|
||||
(entry["event"], entry["timestamp"]): entry["data"]
|
||||
for entry in m.timeline
|
||||
}
|
||||
# Alice's session is keyed by LID but labelled with the phone number
|
||||
# resolved through LID.sqlite.
|
||||
assert events[("chat_first_message", "2025-08-27 15:06:40.000000")] == (
|
||||
"First stored message in WhatsApp chat with "
|
||||
"'Alice Example' (+14155550100)"
|
||||
)
|
||||
assert events[("chat_last_message", "2025-08-28 18:53:20.000000")] == (
|
||||
"Last message in WhatsApp chat with "
|
||||
"'Alice Example' (+14155550100)"
|
||||
)
|
||||
assert events[("lid_pair_recorded", "2025-08-25 07:33:20.000000")] == (
|
||||
"WhatsApp associated LID 100000000000001 with "
|
||||
"phone number 14155550100"
|
||||
)
|
||||
assert events[("group_created", "2025-08-21 20:13:20.000000")] == (
|
||||
"WhatsApp group chat 'Example Group' "
|
||||
"(120000000000000001@g.us) was created"
|
||||
)
|
||||
assert ("chat_first_message", "2025-08-29 22:40:00.000000") in events
|
||||
assert ("chat_last_message", "2025-08-31 02:26:40.000000") in events
|
||||
|
||||
assert len(m.alertstore.alerts) == 0
|
||||
|
||||
|
||||
def test_collect_url_results_includes_expansion():
|
||||
module = Whatsapp(
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
from mvt.common.module import run_module
|
||||
from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts
|
||||
|
||||
from ..utils import get_ios_backup_folder
|
||||
|
||||
|
||||
class TestWhatsappContactsModule:
|
||||
def test_extraction(self):
|
||||
m = WhatsappContacts(target_path=get_ios_backup_folder())
|
||||
run_module(m)
|
||||
assert len(m.results) == 2
|
||||
|
||||
alice = next(r for r in m.results if r["given_name"] == "Alice")
|
||||
assert alice["full_name"] == "Alice Example"
|
||||
assert alice["phone_number"] == "+14155550100"
|
||||
assert alice["whatsapp_id"] == "14155550100@s.whatsapp.net"
|
||||
assert alice["lid"] == "100000000000001@lid"
|
||||
assert alice["user_name"] == "alice.example"
|
||||
assert alice["disappearing_mode_duration"] == 86400.0
|
||||
assert alice["disappearing_mode_is_on"] is True
|
||||
assert alice["disappearing_mode_label"] == "24 hours"
|
||||
assert alice["disappearing_mode_timestamp"] == "2025-07-23 21:46:40.000000"
|
||||
assert alice["about_timestamp"] == "2025-07-12 08:00:00.000000"
|
||||
assert alice["about_expiration_timestamp"] == "2025-08-16 01:20:00.000000"
|
||||
assert alice["last_updated"] == "2025-08-04 11:33:20.000000"
|
||||
|
||||
bob = next(r for r in m.results if r["given_name"] == "Bob")
|
||||
assert bob["lid"] is None
|
||||
assert bob["disappearing_mode_duration"] is None
|
||||
assert bob["disappearing_mode_is_on"] is False
|
||||
assert bob["disappearing_mode_label"] == "off"
|
||||
assert bob["disappearing_mode_timestamp"] is None
|
||||
|
||||
# Alice: disappearing_mode_set, about_changed, about_expiration and
|
||||
# contact_last_updated. Bob: contact_last_updated only.
|
||||
assert len(m.timeline) == 5
|
||||
|
||||
events = {
|
||||
(entry["event"], entry["timestamp"]): entry["data"]
|
||||
for entry in m.timeline
|
||||
}
|
||||
assert (
|
||||
"24 hours"
|
||||
in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")]
|
||||
)
|
||||
assert (
|
||||
"14155550100@s.whatsapp.net (Alice Example)"
|
||||
in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")]
|
||||
)
|
||||
assert (
|
||||
'changed to "Hey there! I am using WhatsApp."'
|
||||
in events[("about_changed", "2025-07-12 08:00:00.000000")]
|
||||
)
|
||||
assert (
|
||||
"scheduled to expire"
|
||||
in events[("about_expiration", "2025-08-16 01:20:00.000000")]
|
||||
)
|
||||
|
||||
updated = [
|
||||
entry["data"]
|
||||
for entry in m.timeline
|
||||
if entry["event"] == "contact_last_updated"
|
||||
]
|
||||
assert len(updated) == 2
|
||||
assert all(
|
||||
entry["timestamp"] == "2025-08-04 11:33:20.000000"
|
||||
for entry in m.timeline
|
||||
if entry["event"] == "contact_last_updated"
|
||||
)
|
||||
assert any("14155550101@s.whatsapp.net (Bob Example)" in d for d in updated)
|
||||
|
||||
assert len(m.alertstore.alerts) == 0
|
||||
|
||||
def test_missing_database(self, tmp_path):
|
||||
m = WhatsappContacts(target_path=str(tmp_path))
|
||||
run_module(m)
|
||||
assert m.results == []
|
||||
assert len(m.alertstore.alerts) == 0
|
||||
@@ -15,8 +15,8 @@ class TestFilesystem:
|
||||
def test_filesystem(self):
|
||||
m = Filesystem(target_path=get_ios_backup_folder())
|
||||
run_module(m)
|
||||
assert len(m.results) == 15
|
||||
assert len(m.timeline) == 15
|
||||
assert len(m.results) == 23
|
||||
assert len(m.timeline) == 23
|
||||
assert len(m.alertstore.alerts) == 0
|
||||
|
||||
def test_detection(self, indicator_file):
|
||||
@@ -29,6 +29,6 @@ class TestFilesystem:
|
||||
)
|
||||
m.indicators = ind
|
||||
run_module(m)
|
||||
assert len(m.results) == 15
|
||||
assert len(m.timeline) == 15
|
||||
assert len(m.results) == 23
|
||||
assert len(m.timeline) == 23
|
||||
assert len(m.alertstore.alerts) == 1
|
||||
@@ -0,0 +1,85 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
"""Helpers building throwaway plugin distributions for the tests.
|
||||
|
||||
Some plugin behaviour only shows up in a fresh interpreter: what an import
|
||||
executes, and what a plugin sees when MVT is imported before or after it.
|
||||
These helpers write an importable distribution with a real entry point and
|
||||
run a script against it in a subprocess, with a temporary home so that the
|
||||
subprocess cannot touch the configuration of whoever runs the tests.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
FIXTURE_COMMAND_NAME = "fixture-plugin"
|
||||
FIXTURE_MODULE_NAME = "fixture_cli_plugin"
|
||||
FIXTURE_DISTRIBUTION_NAME = "fixture-cli-plugin"
|
||||
|
||||
|
||||
def write_cli_plugin_distribution(
|
||||
site_path: Path,
|
||||
entry_point_group: str,
|
||||
module_source: str,
|
||||
) -> Path:
|
||||
"""Write a distribution registering a CLI plugin entry point.
|
||||
|
||||
:param site_path: Folder to write the distribution into, to be added to
|
||||
the import path of the interpreter loading it.
|
||||
:param entry_point_group: Entry-point group to register the command in.
|
||||
:param module_source: Source of the plugin module, which must define a
|
||||
Click command named `cli`.
|
||||
:returns: The folder the distribution was written to.
|
||||
"""
|
||||
site_path.mkdir(parents=True, exist_ok=True)
|
||||
(site_path / f"{FIXTURE_MODULE_NAME}.py").write_text(
|
||||
module_source, encoding="utf-8"
|
||||
)
|
||||
|
||||
dist_info = (
|
||||
site_path / f"{FIXTURE_DISTRIBUTION_NAME.replace('-', '_')}-1.0.dist-info"
|
||||
)
|
||||
dist_info.mkdir(exist_ok=True)
|
||||
(dist_info / "METADATA").write_text(
|
||||
f"Metadata-Version: 2.1\nName: {FIXTURE_DISTRIBUTION_NAME}\nVersion: 1.0\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(dist_info / "entry_points.txt").write_text(
|
||||
f"[{entry_point_group}]\n{FIXTURE_COMMAND_NAME} = {FIXTURE_MODULE_NAME}:cli\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return site_path
|
||||
|
||||
|
||||
def run_isolated_python(
|
||||
script: str,
|
||||
home: Path,
|
||||
site_path: Optional[Path] = None,
|
||||
**environment: str,
|
||||
) -> subprocess.CompletedProcess:
|
||||
"""Run a script in a fresh interpreter with its own configuration folder.
|
||||
|
||||
Importing MVT writes its configuration file, so the subprocess gets a
|
||||
temporary home and no MVT environment variables from the test session.
|
||||
"""
|
||||
isolated_environment = {
|
||||
key: value for key, value in os.environ.items() if not key.startswith("MVT_")
|
||||
}
|
||||
isolated_environment["HOME"] = str(home)
|
||||
isolated_environment["XDG_CONFIG_HOME"] = str(home / "config")
|
||||
if site_path is not None:
|
||||
isolated_environment["PYTHONPATH"] = str(site_path)
|
||||
isolated_environment.update(environment)
|
||||
|
||||
return subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=isolated_environment,
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from mvt.cli import cli
|
||||
from mvt.common.updates import IndicatorsUpdates
|
||||
from mvt.common.version import MVT_VERSION
|
||||
|
||||
# Keep the banner of the group callback from checking for updates online.
|
||||
OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"]
|
||||
|
||||
|
||||
class TestMvtCommand:
|
||||
def test_running_mvt_alone_shows_the_logo_and_the_commands(self):
|
||||
result = CliRunner().invoke(cli, OFFLINE)
|
||||
|
||||
assert result.exit_code == 0
|
||||
logo_at = result.output.index("Mobile Verification Toolkit")
|
||||
usage_at = result.output.index("Usage:")
|
||||
assert logo_at < usage_at
|
||||
assert "mvt-ios" in result.output and "mvt-android" in result.output
|
||||
|
||||
def test_help_reminds_where_the_analysis_runs(self):
|
||||
result = CliRunner().invoke(cli, ["--help"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "mvt-ios" in result.output
|
||||
assert "mvt-android" in result.output
|
||||
assert "check-*" in result.output
|
||||
|
||||
def test_version_prints_the_installed_version(self):
|
||||
result = CliRunner().invoke(cli, [*OFFLINE, "version"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert f"Version: {MVT_VERSION}" in result.output
|
||||
|
||||
def test_download_iocs_updates_the_indicators(self, monkeypatch):
|
||||
updates = []
|
||||
monkeypatch.setattr(
|
||||
IndicatorsUpdates, "update", lambda self: updates.append(self)
|
||||
)
|
||||
|
||||
result = CliRunner().invoke(cli, [*OFFLINE, "download-iocs"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert len(updates) == 1
|
||||
@@ -0,0 +1,280 @@
|
||||
# Mobile Verification Toolkit (MVT)
|
||||
# Copyright (c) 2021-2026 The MVT Authors.
|
||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||
# https://license.mvt.re/1.1/
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import click
|
||||
import pytest
|
||||
|
||||
import mvt.android
|
||||
import mvt.cli
|
||||
import mvt.ios
|
||||
from mvt.android.cli import cli as android_cli
|
||||
from mvt.android.cli import main as android_main
|
||||
from mvt.cli import cli as mvt_cli
|
||||
from mvt.cli import main as mvt_main
|
||||
from mvt.common.cli_plugins import (
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
IOS_CLI_PLUGIN_GROUP,
|
||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||
MVT_CUSTOM_COMMANDS_ENV,
|
||||
MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
)
|
||||
from mvt.ios.cli import cli as ios_cli
|
||||
from mvt.ios.cli import main as ios_main
|
||||
|
||||
from .plugin_fixtures import (
|
||||
FIXTURE_COMMAND_NAME,
|
||||
run_isolated_python,
|
||||
write_cli_plugin_distribution,
|
||||
)
|
||||
|
||||
MARKER_PLUGIN_TEMPLATE = """
|
||||
import os
|
||||
|
||||
import click
|
||||
|
||||
# Touched when this module is imported, so a test can tell whether loading MVT
|
||||
# executed the plugin.
|
||||
open(os.environ["FIXTURE_PLUGIN_MARKER"], "a").close()
|
||||
|
||||
|
||||
@click.command()
|
||||
def cli():
|
||||
click.echo("fixture plugin ran")
|
||||
"""
|
||||
|
||||
PROGRAMS = {
|
||||
"mvt": (mvt.cli, mvt_cli, NEUTRAL_CLI_PLUGIN_GROUP, MVT_CUSTOM_COMMANDS_ENV),
|
||||
"mvt-ios": (mvt.ios, ios_cli, IOS_CLI_PLUGIN_GROUP, MVT_IOS_CUSTOM_COMMANDS_ENV),
|
||||
"mvt-android": (
|
||||
mvt.android,
|
||||
android_cli,
|
||||
ANDROID_CLI_PLUGIN_GROUP,
|
||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||
),
|
||||
}
|
||||
|
||||
CASE_SUMMARY_COMMAND = """
|
||||
import click
|
||||
|
||||
|
||||
@click.command("case-summary")
|
||||
def cli():
|
||||
click.echo("case summary ran")
|
||||
"""
|
||||
|
||||
# The entry-point group of another program, for each program: no group may add
|
||||
# its commands to a CLI other than its own.
|
||||
OTHER_PROGRAMS_GROUP = {
|
||||
"mvt": IOS_CLI_PLUGIN_GROUP,
|
||||
"mvt-ios": NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
"mvt-android": NEUTRAL_CLI_PLUGIN_GROUP,
|
||||
}
|
||||
|
||||
|
||||
def _install_fixture_entry_point(monkeypatch, entry_point_group, command):
|
||||
def entry_points(*, group):
|
||||
if group != entry_point_group:
|
||||
return []
|
||||
return [
|
||||
SimpleNamespace(
|
||||
name=FIXTURE_COMMAND_NAME,
|
||||
value="fixture_cli_plugin:cli",
|
||||
load=lambda: command,
|
||||
dist=SimpleNamespace(
|
||||
metadata={"Name": "fixture-cli-plugin"}, version="1.0"
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"mvt.common.cli_plugins.importlib.metadata.entry_points", entry_points
|
||||
)
|
||||
|
||||
|
||||
def _offline_argv(program, *arguments):
|
||||
"""Build an argument list which keeps the CLI from checking for updates."""
|
||||
return [
|
||||
program,
|
||||
"--disable-update-check",
|
||||
"--disable-indicator-update-check",
|
||||
*arguments,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("program", sorted(PROGRAMS))
|
||||
def test_main_registers_installed_plugins_before_running_the_cli(
|
||||
program, monkeypatch, capsys, restore_cli_commands
|
||||
):
|
||||
package, group, entry_point_group, _ = PROGRAMS[program]
|
||||
|
||||
@click.command()
|
||||
def fixture_command():
|
||||
click.echo("fixture plugin ran")
|
||||
|
||||
_install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command)
|
||||
monkeypatch.setattr(sys, "argv", _offline_argv(program, FIXTURE_COMMAND_NAME))
|
||||
|
||||
with pytest.raises(SystemExit) as exit_info:
|
||||
package.main()
|
||||
|
||||
assert exit_info.value.code == 0
|
||||
assert "fixture plugin ran" in capsys.readouterr().out
|
||||
assert FIXTURE_COMMAND_NAME in group.commands
|
||||
|
||||
|
||||
@pytest.mark.parametrize("program", sorted(PROGRAMS))
|
||||
def test_main_completes_plugin_command_names(
|
||||
program, monkeypatch, capsys, restore_cli_commands
|
||||
):
|
||||
package, _, entry_point_group, _ = PROGRAMS[program]
|
||||
|
||||
@click.command()
|
||||
def fixture_command():
|
||||
pass
|
||||
|
||||
_install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command)
|
||||
complete_variable = f"_{program.upper().replace('-', '_')}_COMPLETE"
|
||||
monkeypatch.setenv(complete_variable, "bash_complete")
|
||||
monkeypatch.setenv("COMP_WORDS", f"{program} fixture")
|
||||
monkeypatch.setenv("COMP_CWORD", "1")
|
||||
monkeypatch.setattr(sys, "argv", [program])
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
package.main()
|
||||
|
||||
assert f"plain,{FIXTURE_COMMAND_NAME}" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("program", sorted(PROGRAMS))
|
||||
def test_main_still_loads_commands_from_a_file(
|
||||
program, monkeypatch, capsys, tmp_path, restore_cli_commands
|
||||
):
|
||||
package, _, entry_point_group, _ = PROGRAMS[program]
|
||||
command_path = tmp_path / "case_summary.py"
|
||||
command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8")
|
||||
_install_fixture_entry_point(
|
||||
monkeypatch, entry_point_group, click.Command("unused")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
_offline_argv(program, "--load-command", str(command_path), "case-summary"),
|
||||
)
|
||||
|
||||
with pytest.raises(SystemExit) as exit_info:
|
||||
package.main()
|
||||
|
||||
assert exit_info.value.code == 0
|
||||
assert "case summary ran" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("program", sorted(PROGRAMS))
|
||||
def test_main_loads_commands_from_the_environment_variable(
|
||||
program, monkeypatch, capsys, tmp_path, restore_cli_commands
|
||||
):
|
||||
# Each CLI reads its own variable, so a main() reading another CLI's would
|
||||
# go unnoticed without this.
|
||||
package, _, _, environment_variable = PROGRAMS[program]
|
||||
command_path = tmp_path / "case_summary.py"
|
||||
command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8")
|
||||
monkeypatch.setenv(environment_variable, str(command_path))
|
||||
monkeypatch.setattr(sys, "argv", _offline_argv(program, "case-summary"))
|
||||
|
||||
with pytest.raises(SystemExit) as exit_info:
|
||||
package.main()
|
||||
|
||||
assert exit_info.value.code == 0
|
||||
assert "case summary ran" in capsys.readouterr().out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("program", sorted(PROGRAMS))
|
||||
def test_main_ignores_the_entry_point_groups_of_the_other_programs(
|
||||
program, monkeypatch, capsys, restore_cli_commands
|
||||
):
|
||||
package, group, _, _ = PROGRAMS[program]
|
||||
_install_fixture_entry_point(
|
||||
monkeypatch,
|
||||
OTHER_PROGRAMS_GROUP[program],
|
||||
click.Command(FIXTURE_COMMAND_NAME),
|
||||
)
|
||||
monkeypatch.setattr(sys, "argv", _offline_argv(program, "--help"))
|
||||
|
||||
with pytest.raises(SystemExit) as exit_info:
|
||||
package.main()
|
||||
|
||||
assert exit_info.value.code == 0
|
||||
assert FIXTURE_COMMAND_NAME not in group.commands
|
||||
assert FIXTURE_COMMAND_NAME not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_the_console_script_targets_are_importable():
|
||||
# [project.scripts] points at these, so they must stay where they are.
|
||||
assert mvt.cli.main is mvt_main
|
||||
assert mvt.ios.main is ios_main
|
||||
assert mvt.android.main is android_main
|
||||
|
||||
|
||||
def test_importing_mvt_does_not_import_a_cli(tmp_path):
|
||||
# The mvt package deliberately re-exports nothing of mvt.cli, so that
|
||||
# importing MVT stays cheap and free of side effects.
|
||||
result = run_isolated_python(
|
||||
"import sys\n"
|
||||
"import mvt\n"
|
||||
"print('imported a cli' if 'mvt.cli' in sys.modules else 'imported mvt')\n",
|
||||
home=tmp_path / "home",
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert result.stdout.strip() == "imported mvt"
|
||||
|
||||
|
||||
def test_importing_mvt_does_not_run_installed_plugins(tmp_path):
|
||||
site_path = write_cli_plugin_distribution(
|
||||
tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE
|
||||
)
|
||||
marker = tmp_path / "plugin-imported"
|
||||
|
||||
result = run_isolated_python(
|
||||
"import mvt.ios.cli\nimport mvt.android.cli\nprint('imported')",
|
||||
home=tmp_path / "home",
|
||||
site_path=site_path,
|
||||
FIXTURE_PLUGIN_MARKER=str(marker),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "imported" in result.stdout
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_registering_the_plugins_runs_the_entry_point(tmp_path):
|
||||
site_path = write_cli_plugin_distribution(
|
||||
tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE
|
||||
)
|
||||
marker = tmp_path / "plugin-imported"
|
||||
|
||||
result = run_isolated_python(
|
||||
"import click\n"
|
||||
"from mvt.common.cli_plugins import (\n"
|
||||
" IOS_CLI_PLUGIN_GROUP,\n"
|
||||
" BrokenPluginCommand,\n"
|
||||
" register_installed_cli_commands,\n"
|
||||
")\n"
|
||||
"group = click.Group()\n"
|
||||
"register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)\n"
|
||||
f"command = group.commands[{FIXTURE_COMMAND_NAME!r}]\n"
|
||||
"assert not isinstance(command, BrokenPluginCommand), command.help\n"
|
||||
"print('registered')\n",
|
||||
home=tmp_path / "home",
|
||||
site_path=site_path,
|
||||
FIXTURE_PLUGIN_MARKER=str(marker),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "registered" in result.stdout
|
||||
assert marker.exists()
|
||||
@@ -1,5 +1,7 @@
|
||||
import io
|
||||
import tarfile
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from mvt.ios.cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
|
||||
from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
|
||||
@@ -70,6 +72,46 @@ def test_check_sysdiagnose_from_archive_closes_archive(tmp_path):
|
||||
{"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
|
||||
]
|
||||
assert command.executed[0].ips_files == [
|
||||
{"file_path": "sysdiagnose/report.ips", "bug_type": 210}
|
||||
{
|
||||
"file_path": str(
|
||||
Path(command.extracted_sysdiagnose_path) / "report.ips"
|
||||
),
|
||||
"bug_type": 210,
|
||||
}
|
||||
]
|
||||
assert command.sysdiagnose_archive is None
|
||||
|
||||
|
||||
def test_archive_is_extracted_once_and_unsafe_members_are_skipped(tmp_path):
|
||||
archive_path = tmp_path / "sysdiagnose.tar.gz"
|
||||
escaped_path = tmp_path / "escaped.txt"
|
||||
content = b"test content"
|
||||
member = tarfile.TarInfo("sysdiagnose/artifact.txt")
|
||||
member.size = len(content)
|
||||
|
||||
with tarfile.open(archive_path, "w:gz") as archive:
|
||||
archive.addfile(member, io.BytesIO(content))
|
||||
escaped = tarfile.TarInfo(f"sysdiagnose/../../{escaped_path.name}")
|
||||
escaped.size = len(content)
|
||||
archive.addfile(escaped, io.BytesIO(content))
|
||||
link = tarfile.TarInfo("sysdiagnose/link")
|
||||
link.type = tarfile.SYMTYPE
|
||||
link.linkname = "/etc/hostname"
|
||||
archive.addfile(link)
|
||||
|
||||
command = CmdIOSCheckSysdiagnose(target_path=str(archive_path))
|
||||
try:
|
||||
command.init()
|
||||
extracted_path = Path(command.extracted_sysdiagnose_path)
|
||||
assert (extracted_path / "artifact.txt").read_bytes() == content
|
||||
assert not escaped_path.exists()
|
||||
assert not (extracted_path / "link").exists()
|
||||
|
||||
module = SysdiagnoseExtraction()
|
||||
command.module_init(module)
|
||||
assert module.tar is None
|
||||
assert module.parent_path == str(extracted_path.parent)
|
||||
finally:
|
||||
command.finish()
|
||||
|
||||
assert not extracted_path.exists()
|
||||
+43
-23
@@ -6,56 +6,60 @@
|
||||
from click.testing import CliRunner
|
||||
|
||||
from mvt.android.cli import cli as android_cli
|
||||
from mvt.cli import cli as mvt_cli
|
||||
from mvt.ios.cli import cli as ios_cli
|
||||
|
||||
|
||||
class TestCompletionCommand:
|
||||
def test_completion_prints_instructions_by_default(self):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(ios_cli, ["completion"])
|
||||
result = runner.invoke(mvt_cli, ["completion"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Shell completion for mvt-ios" in result.output
|
||||
assert "mvt-ios completion bash > ~/.mvt-ios-complete.bash" in result.output
|
||||
assert "Shell completion for mvt, mvt-ios and mvt-android" in result.output
|
||||
assert "mvt completion bash > ~/.mvt-complete.bash" in result.output
|
||||
assert "Mobile Verification Toolkit" not in result.output
|
||||
|
||||
def test_completion_prints_bash_script(self):
|
||||
def test_completion_bash_script_covers_every_cli(self):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(ios_cli, ["completion", "bash"])
|
||||
result = runner.invoke(mvt_cli, ["completion", "bash"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "_MVT_COMPLETE=bash_complete" in result.output
|
||||
assert "_MVT_IOS_COMPLETE=bash_complete" in result.output
|
||||
assert "_MVT_ANDROID_COMPLETE=bash_complete" in result.output
|
||||
assert "complete -o nosort" in result.output
|
||||
assert "mvt-ios" in result.output
|
||||
assert "Mobile Verification Toolkit" not in result.output
|
||||
|
||||
def test_completion_prints_fish_script(self):
|
||||
def test_completion_fish_script_covers_every_cli(self):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(android_cli, ["completion", "fish"])
|
||||
result = runner.invoke(mvt_cli, ["completion", "fish"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "_MVT_ANDROID_COMPLETE=fish_complete" in result.output
|
||||
assert "complete --no-files --command mvt-ios" in result.output
|
||||
assert "complete --no-files --command mvt-android" in result.output
|
||||
assert "complete --no-files --command mvt " in result.output
|
||||
assert "Mobile Verification Toolkit" not in result.output
|
||||
|
||||
def test_completion_install_updates_bashrc_once(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(ios_cli, ["completion", "bash", "--install"])
|
||||
result = runner.invoke(mvt_cli, ["completion", "bash", "--install"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
script_path = tmp_path / ".mvt-ios-complete.bash"
|
||||
script_path = tmp_path / ".mvt-complete.bash"
|
||||
bashrc_path = tmp_path / ".bashrc"
|
||||
assert script_path.exists()
|
||||
assert "_MVT_IOS_COMPLETE=bash_complete" in script_path.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
script = script_path.read_text(encoding="utf-8")
|
||||
assert "_MVT_COMPLETE=bash_complete" in script
|
||||
assert "_MVT_IOS_COMPLETE=bash_complete" in script
|
||||
assert "_MVT_ANDROID_COMPLETE=bash_complete" in script
|
||||
bashrc = bashrc_path.read_text(encoding="utf-8")
|
||||
assert "[ -f" in bashrc
|
||||
assert ".mvt-ios-complete.bash" in bashrc
|
||||
assert ".mvt-complete.bash" in bashrc
|
||||
|
||||
result = runner.invoke(ios_cli, ["completion", "bash", "--install"])
|
||||
result = runner.invoke(mvt_cli, ["completion", "bash", "--install"])
|
||||
assert result.exit_code == 0
|
||||
assert bashrc_path.read_text(encoding="utf-8") == bashrc
|
||||
|
||||
@@ -65,14 +69,30 @@ class TestCompletionCommand:
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
runner = CliRunner()
|
||||
|
||||
result = runner.invoke(android_cli, ["completion", "fish", "--install"])
|
||||
result = runner.invoke(mvt_cli, ["completion", "fish", "--install"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
script_path = (
|
||||
tmp_path / ".config" / "fish" / "completions" / "mvt-android.fish"
|
||||
)
|
||||
script_path = tmp_path / ".config" / "fish" / "conf.d" / "mvt-completion.fish"
|
||||
assert script_path.exists()
|
||||
assert "_MVT_ANDROID_COMPLETE=fish_complete" in script_path.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
script = script_path.read_text(encoding="utf-8")
|
||||
assert "_MVT_COMPLETE=fish_complete" in script
|
||||
assert "_MVT_IOS_COMPLETE=fish_complete" in script
|
||||
assert "_MVT_ANDROID_COMPLETE=fish_complete" in script
|
||||
assert not (tmp_path / ".fishrc").exists()
|
||||
assert not (tmp_path / ".bashrc").exists()
|
||||
assert not (tmp_path / ".zshrc").exists()
|
||||
|
||||
def test_completion_install_without_shell_is_a_usage_error(self):
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(mvt_cli, ["completion", "--install"])
|
||||
|
||||
assert result.exit_code == 2
|
||||
assert "A shell is required when using --install." in result.output
|
||||
|
||||
def test_completion_is_not_a_command_of_the_platform_clis(self):
|
||||
runner = CliRunner()
|
||||
|
||||
assert "completion" not in ios_cli.commands
|
||||
assert "completion" not in android_cli.commands
|
||||
assert runner.invoke(ios_cli, ["completion"]).exit_code == 2
|
||||
assert runner.invoke(android_cli, ["completion"]).exit_code == 2
|
||||
@@ -1,3 +1,7 @@
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from mvt.android.cli import check_bugreport
|
||||
@@ -5,7 +9,9 @@ from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF
|
||||
from mvt.android.cmd_check_backup import CmdAndroidCheckBackup
|
||||
from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport
|
||||
from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
|
||||
from mvt.common import module_loader
|
||||
from mvt.common.module import MVTModule
|
||||
from mvt.common.version import MVT_VERSION
|
||||
from mvt.ios.cli import check_backup, check_fs
|
||||
|
||||
|
||||
@@ -106,6 +112,150 @@ def test_custom_modules_load_from_environment_without_cli_flag(tmp_path, monkeyp
|
||||
assert "EnvBugreportModule" in result.output
|
||||
|
||||
|
||||
class InstalledPackageModule(MVTModule):
|
||||
supported_commands = (("ios", "check-backup"),)
|
||||
|
||||
|
||||
def get_installed_package_modules():
|
||||
return [InstalledPackageModule]
|
||||
|
||||
|
||||
def _fake_entry_points(monkeypatch, value, name="test-modules"):
|
||||
entry_point = importlib.metadata.EntryPoint(
|
||||
name=name, value=value, group=module_loader.MODULES_ENTRY_POINT_GROUP
|
||||
)
|
||||
|
||||
def fake_entry_points(*, group):
|
||||
assert group == module_loader.MODULES_ENTRY_POINT_GROUP
|
||||
return [entry_point]
|
||||
|
||||
monkeypatch.setattr(
|
||||
module_loader.importlib.metadata, "entry_points", fake_entry_points
|
||||
)
|
||||
|
||||
|
||||
def test_installed_module_package_loads_from_entry_point(monkeypatch):
|
||||
_fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules")
|
||||
|
||||
modules = module_loader.load_custom_modules()
|
||||
|
||||
assert modules == [InstalledPackageModule]
|
||||
|
||||
|
||||
def test_broken_module_entry_point_is_skipped(monkeypatch, caplog):
|
||||
_fake_entry_points(monkeypatch, "nonexistent_module_xyz:get_modules")
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
modules = module_loader.load_custom_modules()
|
||||
|
||||
assert modules == []
|
||||
assert "Unable to load modules from entry point" in caplog.text
|
||||
|
||||
|
||||
def test_entry_point_module_deduplicated_against_paths(monkeypatch, tmp_path):
|
||||
_fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules")
|
||||
module_path = _write_custom_module(
|
||||
tmp_path / "custom.py",
|
||||
"PathLoadedModule",
|
||||
(("ios", "check-backup"),),
|
||||
)
|
||||
|
||||
modules = module_loader.load_custom_modules([str(module_path)])
|
||||
|
||||
assert [module.__name__ for module in modules] == [
|
||||
"InstalledPackageModule",
|
||||
"PathLoadedModule",
|
||||
]
|
||||
|
||||
|
||||
def test_list_modules_shows_module_sources(tmp_path, caplog):
|
||||
module_path = _write_custom_module(
|
||||
tmp_path / "custom.py",
|
||||
"SourcedBackupModule",
|
||||
(("ios", "check-backup"),),
|
||||
)
|
||||
file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest()
|
||||
custom_modules = module_loader.load_custom_modules([str(module_path)])
|
||||
|
||||
from mvt.ios.cmd_check_backup import CmdIOSCheckBackup
|
||||
|
||||
cmd = CmdIOSCheckBackup(target_path=str(tmp_path), custom_modules=custom_modules)
|
||||
cmd.list_modules()
|
||||
|
||||
assert f" - Modules from 'mvt@{MVT_VERSION}':" in caplog.text
|
||||
assert (
|
||||
f" - Modules from '{module_path}' (sha256: {file_sha256}): SourcedBackupModule"
|
||||
in caplog.text
|
||||
)
|
||||
|
||||
|
||||
def test_builtin_module_origin():
|
||||
from mvt.ios.modules.backup import BACKUP_MODULES
|
||||
|
||||
origin = module_loader.get_module_origin(BACKUP_MODULES[0])
|
||||
|
||||
assert origin.kind == "builtin"
|
||||
assert origin.name == "mvt"
|
||||
assert origin.version == MVT_VERSION
|
||||
|
||||
|
||||
def test_installed_module_origin(monkeypatch):
|
||||
_fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules")
|
||||
|
||||
modules = module_loader.load_custom_modules()
|
||||
|
||||
origin = module_loader.get_module_origin(modules[0])
|
||||
assert origin.kind == "package"
|
||||
assert origin.name == "test-modules"
|
||||
|
||||
|
||||
def test_distribution_commit_read_from_direct_url():
|
||||
class FakeDistribution:
|
||||
def read_text(self, filename):
|
||||
assert filename == "direct_url.json"
|
||||
return json.dumps(
|
||||
{
|
||||
"url": "https://github.com/example/example-modules",
|
||||
"vcs_info": {"commit_id": "abc1234", "vcs": "git"},
|
||||
}
|
||||
)
|
||||
|
||||
assert module_loader._distribution_commit(FakeDistribution()) == "abc1234"
|
||||
|
||||
|
||||
def test_command_log_records_loaded_modules(tmp_path):
|
||||
(tmp_path / "Manifest.db").touch()
|
||||
(tmp_path / "Info.plist").touch()
|
||||
module_path = _write_custom_module(
|
||||
tmp_path / "custom.py",
|
||||
"AuditedRunModule",
|
||||
(("ios", "check-backup"),),
|
||||
slug="audited_run_module",
|
||||
)
|
||||
file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest()
|
||||
output_path = tmp_path / "out"
|
||||
|
||||
result = CliRunner().invoke(
|
||||
check_backup,
|
||||
[
|
||||
"--module",
|
||||
"AuditedRunModule",
|
||||
"--load-module",
|
||||
str(module_path),
|
||||
"--output",
|
||||
str(output_path),
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
command_log = (output_path / "command.log").read_text(encoding="utf-8")
|
||||
assert (
|
||||
f"Loaded 1 check-backup modules from '{module_path}' "
|
||||
f"(sha256: {file_sha256}): AuditedRunModule" in command_log
|
||||
)
|
||||
|
||||
|
||||
class NestedBugreportModule(MVTModule):
|
||||
supported_commands = (("android", "check-bugreport"),)
|
||||
|
||||
|
||||
Reference in new issue
Block a user