diff --git a/.github/workflows/publish-release-docker.yml b/.github/workflows/publish-release-docker.yml index eddca22..a90202d 100644 --- a/.github/workflows/publish-release-docker.yml +++ b/.github/workflows/publish-release-docker.yml @@ -37,6 +37,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + fetch-depth: 0 # the package version comes from the git tags # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 @@ -59,6 +61,9 @@ jobs: type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }} type=sha,suffix=${{ matrix.platform.tag-suffix }} type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }} + # Register emulators so Buildx can build the ARM64 images on the AMD64 runner. + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 # This step sets up some additional capabilities to generate the provenance and sbom attestations - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -71,6 +76,7 @@ jobs: with: file: ${{ matrix.platform.dockerfile }} context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..3c20dc7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + schedule: + - cron: "0 9 * * 1" # Mondays, 09:00 UTC + workflow_dispatch: + push: + tags: ["v*"] + +jobs: + tag: + if: github.ref_type != 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - run: | + last=$(git describe --tags --abbrev=0 --match 'v*') + if [ "$(git rev-list "$last..HEAD" --count)" = 0 ]; then + echo "Nothing merged since $last"; exit 0 + fi + tag=v$(date -u +%Y.%-m.%-d) + gh release create "$tag" --target "$GITHUB_SHA" --generate-notes + # Events made with GITHUB_TOKEN don't start other workflows; run them by hand. + gh workflow run release.yml --ref "$tag" + gh workflow run publish-release-docker.yml --ref "$tag" + + publish: + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + id-token: write # PyPI trusted publishing + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v10.0.0 + - run: uv build && uv publish diff --git a/Dockerfile b/Dockerfile index da06064..138edc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,12 +119,24 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y \ adb \ + binutils \ default-jre-headless \ + file \ + git \ + jq \ + less \ libcurl4 \ + libimage-exiftool-perl \ libssl3 \ libusb-1.0-0 \ + moreutils \ + p7zip-full \ python3 \ - sqlite3 + ripgrep \ + sqlite3 \ + tree \ + unzip \ + xxd COPY --from=build-libplist /build / COPY --from=build-libimobiledevice-glue /build / COPY --from=build-libtatsu /build / @@ -144,7 +156,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2 RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc # Generate adb key folder -RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc +RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc RUN mkdir /root/.android # Setup investigations environment diff --git a/Dockerfile.android b/Dockerfile.android index 2c060db..2b74611 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -31,7 +31,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2 RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc # Generate adb key folder -RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc +RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc RUN mkdir /root/.android ENTRYPOINT [ "/usr/local/bin/mvt-android" ] diff --git a/Makefile b/Makefile index 528c697..6214009 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,3 @@ clean: dist: $(UV) build -upload: - $(UV) tool run twine upload dist/* - -test-upload: - $(UV) tool run twine upload --repository testpypi dist/* diff --git a/README.md b/README.md index 65fcdee..98e90e3 100644 --- a/README.md +++ b/README.md @@ -58,34 +58,33 @@ For alternative installation options and known issues, please refer to the [docu ## Usage -MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) +MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion`, `plugins` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/) + +Pass `--verbose` to any of the three commands, before the command name (`mvt-ios --verbose check-backup ...`), for debug output. The `--verbose` option the `check-*` commands accept after their name still works but is kept for compatibility only and will be removed in a future release. ### Shell completion -MVT can generate shell completion scripts for Bash, Zsh, and Fish: +MVT can generate a shell completion script for Bash, Zsh, and Fish which covers `mvt`, `mvt-ios` and `mvt-android`: ```bash -mvt-ios completion -mvt-android completion +mvt completion ``` -The commands print setup instructions by default. To generate a completion script directly, pass the shell name: +The command prints setup instructions by default. To generate the completion script directly, pass the shell name: ```bash -mvt-ios completion bash -mvt-android completion zsh +mvt completion bash ``` MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details. -Module-running `check-*` commands can load custom Python modules with -`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the -[development documentation](https://docs.mvt.re/en/latest/development/) for -details. -Users can also add top-level commands to `mvt-ios` and `mvt-android` from -installed Python packages or local files and folders. See the -[custom CLI command documentation](https://docs.mvt.re/en/latest/custom_commands/) -for the plugin entry points and `--load-command` interface. +Plugin packages extend MVT with additional forensic modules, which run inside +the `check-*` commands, and with top-level commands on `mvt`, `mvt-ios` and +`mvt-android`. See the +[development documentation](https://docs.mvt.re/en/latest/development/) for +writing and installing them, and the +[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/) +for the entry points a package registers commands in. ## License diff --git a/docs/android/intrusion_logs.md b/docs/android/intrusion_logs.md index da8fc83..16c6bf1 100644 --- a/docs/android/intrusion_logs.md +++ b/docs/android/intrusion_logs.md @@ -58,7 +58,7 @@ mvt-android check-intrusion-logs --output /path/to/results/ /path/to/intrusion-l | `-l, --list-modules` | List the available intrusion-log modules and exit. | | `-m, --module NAME` | Run a single module (e.g. `DnsEvent`) instead of all of them. | | `-t, --timezone TZ` | IANA timezone name for the device (e.g. `Europe/Paris`). When set, event timestamps are converted to the device's local time instead of UTC. | -| `-v, --verbose` | Verbose logging. | +| `-v, --verbose` | Verbose logging. Kept for compatibility and to be removed in a future release: pass `--verbose` to `mvt-android` itself instead. | ## Modules diff --git a/docs/command_completion.md b/docs/command_completion.md index 7204df5..a51cc7a 100644 --- a/docs/command_completion.md +++ b/docs/command_completion.md @@ -6,61 +6,61 @@ Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fi To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using. -The following describes how to generate the command completion scripts and add them to your shell configuration. +`mvt completion` generates one script which covers `mvt`, `mvt-ios` and `mvt-android`. The following describes how to generate that script and add it to your shell configuration. > **Note: You will need to start a new shell for the changes to take effect.** ### For Bash ```bash -# Generate bash completion scripts -mvt-ios completion bash > ~/.mvt-ios-complete.bash -mvt-android completion bash > ~/.mvt-android-complete.bash +# Generate the bash completion script +mvt completion bash > ~/.mvt-complete.bash ``` Add the following to `~/.bashrc`: ```bash -# source mvt completion scripts -[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash -[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash +# source the mvt completion script +[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash ``` ### For Zsh ```bash -# Generate zsh completion scripts -mvt-ios completion zsh > ~/.mvt-ios-complete.zsh -mvt-android completion zsh > ~/.mvt-android-complete.zsh +# Generate the zsh completion script +mvt completion zsh > ~/.mvt-complete.zsh ``` Add the following to `~/.zshrc`: ```bash -# source mvt completion scripts -[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh -[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh +# source the mvt completion script +[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh ``` ### For Fish ```bash -# Generate fish completion scripts -mkdir -p ~/.config/fish/completions -mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish -mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish +# Generate the fish completion script +mkdir -p ~/.config/fish/conf.d +mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish ``` -Fish loads completion files from `~/.config/fish/completions` automatically. +Fish loads the files in `~/.config/fish/conf.d` automatically. ### Automatic Installation MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`: ```bash -mvt-ios completion bash --install -mvt-android completion bash --install +mvt completion bash --install ``` -Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`. +Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/conf.d` and changes no shell configuration. + +!!! note + + Earlier versions generated one script per command, with `mvt-ios completion` + and `mvt-android completion`. Files written by them keep working. When you + switch to the single script, remove the old files and the lines which load + them from your shell configuration. For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion). - diff --git a/docs/custom_commands.md b/docs/custom_commands.md deleted file mode 100644 index ab89fd9..0000000 --- a/docs/custom_commands.md +++ /dev/null @@ -1,107 +0,0 @@ -# Custom CLI Commands - -MVT can load additional top-level commands into `mvt-ios` and `mvt-android`. -Custom commands are different from [custom forensic modules](development.md#custom-modules): -commands add new CLI operations, while modules add analysis steps to existing -`check-*` commands. - -!!! warning - - Custom commands run as trusted Python code inside the MVT process. Install - or load commands only from sources you trust. MVT does not sandbox - third-party commands, and the MVT maintainers do not maintain them. - -## Install a Command Package - -Python packages can register a Click command or group for either MVT CLI. A -minimal package can expose this command from `my_mvt_plugin.py`: - -```python -import click - - -@click.command() -@click.argument("path", type=click.Path(exists=True)) -def summarize(path): - """Summarize an acquisition.""" - click.echo(f"Summarizing {path}") -``` - -Register the object in the package's `pyproject.toml`. The entry-point name is -the command users invoke: - -```toml -[project.entry-points."mvt.ios.cli_plugins"] -summarize = "my_mvt_plugin:summarize" - -[project.entry-points."mvt.android.cli_plugins"] -summarize = "my_mvt_plugin:summarize" -``` - -Use only the iOS or Android group if the command is platform-specific. After -installing the package in the same environment as MVT, it appears directly in -the appropriate CLI: - -```bash -mvt-ios summarize ./ios-backup -mvt-android summarize ./androidqf-output -``` - -For a `pipx` installation of MVT, inject the plugin into MVT's environment: - -```bash -pipx inject mvt my-mvt-plugin -``` - -When MVT is installed in an active virtual environment, install the plugin with -`pip` in that environment. - -## Load a Command File - -For local commands that are not packaged, create a Python file that exports one -Click command or group named `cli`: - -```python -import click - - -@click.command("case-summary") -@click.argument("path", type=click.Path(exists=True)) -def cli(path): - """Summarize a case directory.""" - click.echo(f"Summarizing {path}") -``` - -Pass the file before the custom command name: - -```bash -mvt-ios --load-command ./case_summary.py case-summary ./ios-backup -``` - -`--load-command` can be repeated and also accepts a folder. MVT loads -non-hidden top-level `*.py` files in sorted order and skips `__init__.py`. -Every loaded file must export one `cli` object. - -To load a file or folder on every invocation, set the platform-specific -environment variable: - -```bash -export MVT_IOS_CUSTOM_COMMANDS=./ios_commands -export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands -``` - -## Naming and Errors - -Built-in MVT commands cannot be replaced. External command names must also be -unique; when installed packages or environment paths collide, MVT keeps the -first command and logs a warning. A collision from an explicit -`--load-command` is a usage error. - -A package entry point or environment command that cannot be imported appears -as a marked broken command without preventing other MVT commands from working. -Invoke that command to see its package or file source and the underlying error. -An invalid command supplied explicitly with `--load-command` fails immediately -with a usage error. - -Installed command packages use the entry-point name as the CLI command name. -The entry point must resolve to a `click.Command` or `click.Group`. diff --git a/docs/development.md b/docs/development.md deleted file mode 100644 index ffe1265..0000000 --- a/docs/development.md +++ /dev/null @@ -1,131 +0,0 @@ -# Development - -The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software. - -## Local environment - -MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run: - -```bash -make install -``` - -## Testing - -MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can -be run automatically with: - -```bash -make check -``` - -Run these tests before making new commits or opening pull requests. - -## Module dependencies - -Modules can require other modules to run first by declaring their classes in -`dependencies`. The command runner uses a stable topological ordering, so the -existing module list order is preserved wherever dependency constraints allow. - -```python -class DependentModule(MVTModule): - dependencies = (PrerequisiteModule,) - - def run(self): - prerequisite_results = self.get_dependency_results(PrerequisiteModule) -``` - -Selecting a single module also runs its transitive dependencies. If a dependency -is unavailable or the dependency graph contains a cycle, the command logs a -warning and does not run any modules. - -## Custom modules - -Module-running `check-*` commands can load custom modules from Python files that -are not installed as part of MVT. Load one file with: - -```bash -mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup -``` - -You can also load a folder. MVT loads non-hidden top-level `*.py` files in -sorted order and skips `__init__.py`: - -```bash -mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump -``` - -Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This -folder is loaded before any `--load-module` path: - -```bash -MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip -``` - -Custom modules are normal `MVTModule` subclasses: - -```python -from mvt.common.module import MVTModule - - -class ExampleCustomModule(MVTModule): - supported_commands = (("ios", "check-backup"), ("ios", "check-fs")) - slug = "example_custom_module" - - def run(self): - self.results = [{"message": "custom module ran"}] - - def check_indicators(self): - pass - - def serialize(self, result): - return None -``` - -Use `supported_commands` to declare the platform/command pairs a module -supports. Empty `supported_commands` means the module will not run and MVT logs -a warning. This explicit declaration is required for every command. Supported -pairs are: - -```python -("ios", "check-backup") -("ios", "check-fs") -("ios", "check-iocs") -("ios", "check-sysdiagnose") -("android", "check-backup") -("android", "check-bugreport") -("android", "check-androidqf") -("android", "check-intrusion-logs") -("android", "check-iocs") -``` - -Custom modules can depend on existing MVT module classes. Dependencies are -resolved with the same ordering logic as built-in modules, and custom modules -are appended after built-ins before ordering: - -```python -from mvt.common.module import MVTModule -from mvt.ios.modules.backup.manifest import Manifest - - -class DependentCustomModule(MVTModule): - supported_commands = (("ios", "check-backup"),) - dependencies = (Manifest,) - - def run(self): - manifest_results = self.get_dependency_results(Manifest) - self.results = [{"manifest_entries": len(manifest_results)}] -``` - -## Profiling - -Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be -take to avoid inefficient code paths as we add new modules. - -MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable. - -```bash -MVT_PROFILE=1 dev/mvt-ios check-backup test_backup -``` - -Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT. diff --git a/docs/development/custom_commands.md b/docs/development/custom_commands.md new file mode 100644 index 0000000..7aac490 --- /dev/null +++ b/docs/development/custom_commands.md @@ -0,0 +1,202 @@ +# Custom CLI Commands + +MVT can load additional top-level commands into `mvt`, `mvt-ios` and +`mvt-android`. A command package chooses which of the three each of its +commands is added to. +Custom commands are different from [custom forensic modules](index.md#custom-modules): +commands add new CLI operations, while modules add analysis steps to existing +`check-*` commands. + +!!! warning + + Custom commands run as trusted Python code inside the MVT process. Install + or load commands only from sources you trust. MVT does not sandbox + third-party commands, and the MVT maintainers do not maintain them. + +## Install a Command Package + +Python packages can register a Click command or group on one or more of the MVT CLIs. +A minimal package can expose this command from `my_mvt_plugin.py`: + +```python +import click + + +@click.command() +@click.argument("path", type=click.Path(exists=True)) +def summarize(path): + """Summarize an acquisition.""" + click.echo(f"Summarizing {path}") +``` + +Log through `get_plugin_logger(__name__)` from `mvt.plugin`. Records logged +through `logging.getLogger(__name__)` do not reach `command.log`, and MVT's +console handler does not show them. + +Register the object in the package's `pyproject.toml`. The entry-point name is +the command users invoke: + +```toml +[project.entry-points."mvt.ios.cli_plugins"] +summarize = "my_mvt_plugin:summarize" + +[project.entry-points."mvt.android.cli_plugins"] +summarize = "my_mvt_plugin:summarize" +``` + +Each entry-point group adds the command to one CLI: `mvt.ios.cli_plugins` to +`mvt-ios`, `mvt.android.cli_plugins` to `mvt-android` and `mvt.cli_plugins` to +`mvt`. Register the command in the group of every CLI which should offer it: a +platform-specific command belongs in one platform group, and a command which +handles acquisitions of both platforms, as above, in both. After installing the +package in the same environment as MVT, the command appears directly in those +CLIs: + +```bash +mvt-ios summarize ./ios-backup +mvt-android summarize ./androidqf-output +``` + +For a `pipx` installation of MVT, inject the plugin into MVT's environment: + +```bash +pipx inject mvt my-mvt-plugin +``` + +When MVT is installed in an active virtual environment, install the plugin with +`pip` in that environment. `mvt plugins list` shows the installed packages and +the commands they add, see [Managing Plugins](plugins.md). + +Command packages that need their own settings, such as an API key, should store +them in a namespaced [plugin configuration file](plugin_configuration.md) +rather than in MVT's own `config.yaml`. + +### Commands on `mvt` + +A MVT plugin command can also add sub-commands to the base `mvt` command. This can be used for commands which are not tied to a particular forensic platform: + +```toml +[project.entry-points."mvt.cli_plugins"] +my-plugin = "my_mvt_plugin:my_plugin" +``` + +Commands in this group are added to `mvt` only, so this one is invoked as +`mvt my-plugin`. A command on `mvt` has nothing but its name to say which +plugin it belongs to, so name it after the plugin, and make it a Click group +when the plugin has several operations to offer, such as +`mvt my-plugin configure`. + +## Developing a Command Locally + +A package is how a command is distributed. While a command is being written, +MVT can load it straight from its file instead, so the package need not be +reinstalled after every change; an editable install of the package does the +same through its entry points. Create a Python file that exports one Click +command or group named `cli`: + +```python +import click + + +@click.command("case-summary") +@click.argument("path", type=click.Path(exists=True)) +def cli(path): + """Summarize a case directory.""" + click.echo(f"Summarizing {path}") +``` + +Pass the file before the custom command name: + +```bash +mvt-ios --load-command ./case_summary.py case-summary ./ios-backup +``` + +`--load-command` can be repeated and also accepts a folder. MVT loads +non-hidden top-level `*.py` files in sorted order and skips `__init__.py`. +Every loaded file must export one `cli` object. + +To load a file or folder on every invocation, set the environment variable of +the CLI the commands belong on. Like the entry-point groups, each variable adds +its commands to one CLI only: + +```bash +export MVT_CUSTOM_COMMANDS=./commands +export MVT_IOS_CUSTOM_COMMANDS=./ios_commands +export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands +``` + +## Building a Module-Running Command + +A command which runs forensic modules over an acquisition subclasses `Command`. +`Command` creates the output folder and writes `command.log`. It orders the +modules, resolves their dependencies and runs them. It writes the result files, +`alerts.json` and `info.json`. The subclass sets `platform`, `name` and +`modules`: + +```python +from mvt.plugin import Command, MVTModule, convert_unix_to_iso, get_plugin_logger + +log = get_plugin_logger(__name__) + + +class APKManifest(MVTModule): + supported_commands = (("android", "check-apks"),) + + def run(self): + self.results = [{"checked_at": convert_unix_to_iso(0)}] + + +class CmdCheckAPKs(Command): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.platform = "android" + self.name = "check-apks" + self.modules = [APKManifest] +``` + +`platform` and `name` are the pair the modules declare in +`supported_commands`. `modules` lists the module classes the command runs, +imported directly. A command which runs modules of another plugin depends on +that package in `pyproject.toml` and imports them the same way. `init()`, +`module_init(module)` and `finish()` are optional hooks. `run()` calls them +before the run, before each module and after the run. + +Wrap the command in a Click command with an `--output` option. Run it, then +print the alert summary: + +```python +import click + + +@click.command("check-apks") +@click.option("--output", "-o", type=click.Path(exists=False)) +@click.argument("TARGET_PATH", type=click.Path(exists=True)) +def cli(output, target_path): + cmd = CmdCheckAPKs(target_path=target_path, results_path=output) + log.info("Checking APK files at path: %s", target_path) + cmd.run() + cmd.show_alerts_brief() +``` + +The `--verbose` option of `mvt`, `mvt-ios` and `mvt-android` applies to the +command. The command defines no `--verbose` option of its own. The pair a +plugin command adds is not listed anywhere in MVT. Name it in the plugin's +README. + +## Naming and Errors + +Built-in MVT commands cannot be replaced. External command names must also be +unique on each CLI; when installed packages or environment paths collide, MVT +keeps the first command and logs a warning. The environment path of a CLI is +registered before its installed packages, so a command loaded from there wins a +collision with a package. A collision from an explicit `--load-command` is a +usage error. + +A package entry point or environment command that cannot be imported appears +as a marked broken command without preventing other MVT commands from working. +Invoke that command to see its package or file source and the underlying error. +An invalid command supplied explicitly with `--load-command` fails immediately +with a usage error. + +Installed command packages use the entry-point name as the CLI command name. +The entry point must resolve to a `click.Command` or `click.Group`. diff --git a/docs/development/index.md b/docs/development/index.md new file mode 100644 index 0000000..8b80238 --- /dev/null +++ b/docs/development/index.md @@ -0,0 +1,391 @@ +# Development + +The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software. + +## Local environment + +MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run: + +```bash +make install +``` + +## Testing + +MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can +be run automatically with: + +```bash +make check +``` + +Run these tests before making new commits or opening pull requests. + +## Module dependencies + +Modules can require other modules to run first by declaring their classes in +`dependencies`. The command runner uses a stable topological ordering, so the +existing module list order is preserved wherever dependency constraints allow. + +```python +class DependentModule(MVTModule): + dependencies = (PrerequisiteModule,) + + def run(self): + prerequisite_results = self.get_dependency_results(PrerequisiteModule) +``` + +Selecting a single module also runs its transitive dependencies. + +A module can only depend on modules the command it runs in also has. When a +declared dependency is not among them, the command logs a warning naming the +module and the missing dependency, skips that module and everything depending +on it, and runs the rest of the analysis. Selecting such a module with +`--module` therefore leaves nothing to run, which the warning explains. + +A cycle in the dependency graph is a programming error rather than a +configuration problem: the command logs a warning and runs no modules at all. + +## Custom modules + +MVT's module-running `check-*` commands can run forensic modules which are not +part of MVT. Custom modules are distributed as plugin packages. A Python +package installed next to MVT registers its modules through an entry point. +The modules then load automatically in every command they support, see +[Installed module packages](#installed-module-packages). `mvt plugins list` +shows the installed packages and where each one was installed from. + +MVT can also load module files by path, with `--load-module` and +`MVT_CUSTOM_MODULES`, see +[Developing modules locally](#developing-modules-locally). This can be used +while writing a module. Use a Python package to distribute one. + +A custom module declares in `supported_commands` the platform and command +pairs it runs in. A module with empty `supported_commands` does not run and +MVT logs a warning. The nine pairs are: + +```python +("ios", "check-backup") +("ios", "check-fs") +("ios", "check-iocs") +("ios", "check-sysdiagnose") +("android", "check-backup") +("android", "check-bugreport") +("android", "check-androidqf") +("android", "check-intrusion-logs") +("android", "check-iocs") +``` + +`check-iocs` re-checks stored results rather than an acquisition. It matches +every `.json` file in the results folder to the module with that slug. +It then runs that module's `check_indicators()` again. A module which +implements `check_indicators()` is included in `check-iocs` for its platform. +It does not need to declare the `check-iocs` pair. + +### Writing a module + +A module subclasses `MVTModule` or one of the base classes below and +implements `run()`. `check_indicators()` and `serialize()` are optional. The +first matches results against IOCs or detections. The second returns timeline +records. + +```python +from mvt.plugin import IOSExtraction, convert_unix_to_iso + + +class ExampleCustomModule(IOSExtraction): + supported_commands = ( + ("ios", "check-backup"), + ("ios", "check-fs"), + ) + slug = "example_custom_module" + + def run(self): + self.results = [{"checked_at": convert_unix_to_iso(0)}] + + def check_indicators(self): + pass + + def serialize(self, result): + return None +``` + +The base classes are: + +- `MVTModule`: the base of every module. It provides `self.results`, + `self.alertstore`, `self.log`, `self.indicators` and + `get_dependency_results()`. Subclass it directly for a module which reads + only the results of other modules. +- `IOSExtraction`: `("ios", "check-backup")` and `("ios", "check-fs")`. Adds + `_find_ios_database()`, which locates a module's database in a backup or in + a filesystem dump and repairs it if it is malformed. Adds + `_get_backup_files_from_manifest()`, `_get_backup_file_from_id()` and + `_get_fs_files_from_patterns()`. Adds `_open_sqlite_db()`, which opens a + database read-only. +- `SysdiagnoseExtraction`: `("ios", "check-sysdiagnose")`. MVT extracts the + archive and calls `from_sysdiagnose_folder()` before `run()`. The module + reads files with `_get_files_by_pattern()` and `_get_file_content()`. + `ips_files` lists the crash reports. See + [Check an iOS Sysdiagnose](../ios/sysdiagnose.md). +- `AndroidQFModule`: `("android", "check-androidqf")`. MVT calls `from_dir()` + or `from_zip()` with the file list of the acquisition. The module reads + files with `_get_files_by_pattern()` and `_get_file_content()`. + `_get_device_timezone()` returns the device timezone. +- `AndroidBackupModule`: `("android", "check-backup")`. MVT calls `from_dir()` + or `from_ab()`. The module reads files with `_get_files_by_pattern()` and + `_get_file_content()`. +- `BugReportModule`: `("android", "check-bugreport")`. MVT calls `from_dir()` + or `from_zip()`. The module reads files with `_get_files_by_pattern()`, + `_get_files_by_patterns()` and `_get_file_content()`. + `_get_dumpstate_file()` returns the dumpstate file, and + `_get_file_modification_time()` the modification time of a file. + +The underscore-named helpers are internal to the base classes. Plugin modules +can call them. Their names and signatures can change between releases. Read +the base class in `src/mvt/ios/modules` or `src/mvt/android/modules` before +relying on one. + +### Depending on a built-in module + +A module which post-processes records generated by one or more built-in MVT +modules must declare the source modules in `dependencies`. It reads their +results with `get_dependency_results()`. Import the class from its family +package: `mvt.ios.modules.backup`, `mvt.ios.modules.fs`, +`mvt.ios.modules.mixed`, `mvt.android.modules.androidqf`, +`mvt.android.modules.backup`, `mvt.android.modules.bugreport` or +`mvt.android.modules.intrusion_logs`. + +```python +from mvt.ios.modules.backup import Manifest +from mvt.plugin import MVTModule + + +class DependentCustomModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + dependencies = (Manifest,) + + def run(self): + manifest_results = self.get_dependency_results(Manifest) + self.results = [{"manifest_entries": len(manifest_results)}] +``` + +Dependencies are ordered as for the built-in modules, with custom modules +appended after the built-ins, see [Module dependencies](#module-dependencies). +A dependency has to run in every command the module supports. Where it does +not, MVT skips the module with a warning. + +`get_dependency_results()` returns the plain dictionaries the module produced. +They are the same records it writes to `.json`. Typed results per module +are planned. + +### Replacing a built-in module + +A custom module which extends a built-in one runs alongside it, and when the +two share a slug they write to the same results file. MVT warns about that +collision, naming both modules, where each came from and the file the later one +overwrites, but it runs them both. Set `replaces` to the class the module +supersedes to take that module's place instead: when both are available to a +command, the named module is dropped from the run. + +```python +from mvt.ios.modules.backup import Manifest as BuiltinManifest + + +class Manifest(BuiltinManifest): + supported_commands = (("ios", "check-backup"),) + replaces = BuiltinManifest +``` + +Keeping the class name of the replaced module, as above, keeps the name +`--module` selects the module by and the slug its results file is named after. +A replacement with a different class name writes to a file named after its own +slug, unless it sets `slug` to the slug of the module it replaces, and +`--module` still selects it by the name of the replaced module, which MVT logs. +Taking over the slug of a replaced module is not a collision and is not warned +about, because that module is no longer part of the run. + +Modules which depend on the replaced module receive the replacement instead, so +`get_dependency_results(BuiltinManifest)` returns the replacement's results. A +module must not depend on the module it replaces: that module does not run, so +the replacement has to produce the data itself, and MVT logs a warning when a +module declares both. + +A replacement takes on the obligations of the module it replaces. If it +declares a dependency the command does not provide, it is skipped like any +other module with an unavailable dependency, and the module it replaced stays +out of the run: neither of them produces results, and the modules depending on +the replaced module are skipped as well. + +Subclassing the replaced module is not required, but a replacement which is not +a subclass is logged with a warning, because its results may not be what the +modules depending on the replaced module expect. Naming a module the command +does not run has no effect, and modules which replace each other in a cycle all +keep running and replace nothing. Every applied substitution is logged, so it +is recorded in `command.log` when the command runs with an `--output` folder. + +Every command resolves replacements on its own. `check-iocs` matches stored +results files against the slugs of the modules available for that command. A +replacement which subclasses the module it replaces inherits its +`check_indicators()`. It is then part of `check-iocs` for its platform and +re-checks the results file named after its slug. A replacement with no +`check_indicators()` is not part of `check-iocs`. The built-in module it +replaced re-checks the file. `check-iocs` matches `--module` on the class name +only. Pass a differently named replacement's own name there, not the name of +the module it replaces. + +### Importing from MVT + +Import from `mvt.plugin` if it has what you need. The names it exports are +kept working on a best-effort basis. A change to one of them is announced in +the release notes. Anything else in `mvt` can be imported too, but may change +between releases without notice. The plugin interface is best effort. + +`mvt.plugin` exports the base classes above and `Command`, `Alert` and +`AlertLevel`, the result types, `DatabaseNotFoundError` and +`DatabaseCorruptedError`, the timestamp converters, the settings API of +[Plugin Configuration](plugin_configuration.md), MVT's own `settings`, +`get_plugin_logger()` and `MVT_VERSION`. `src/mvt/plugin.py` holds the list. +Read MVT's `settings` for values such as `NETWORK_ACCESS_ALLOWED` and +`NETWORK_TIMEOUT`. Plugin values go in the plugin's own settings file. + +## Installed module packages + +Python packages can register modules so they load automatically in every +module-running `check-*` command, without `--load-module` or +`MVT_CUSTOM_MODULES`. Register an entry point in the `mvt.modules` group in +the package's `pyproject.toml`: + +```toml +[project.entry-points."mvt.modules"] +mvt-plugin-example-org = "mvt_plugin_example_org:get_modules" +``` + +The entry point must resolve to an iterable of `MVTModule` subclasses, or to +a callable returning one: + +```python +from mvt.plugin import MVTModule + + +class PackagedModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + + def run(self): + self.results = [{"message": "packaged module ran"}] + + +def get_modules() -> list[type[MVTModule]]: + return [PackagedModule] +``` + +`get_modules()` is the package's module list, written by hand. A package which +keeps its modules in separate files imports each class there and lists it. A +module missing from the list does not load. + +Installed modules follow the same rules as other custom modules: each module +must declare `supported_commands`, and dependencies are resolved with the +standard ordering logic. A broken entry point is skipped with a warning and +does not prevent MVT from running. As with custom commands, installed module +packages run as trusted code inside the MVT process, so install only packages +from sources you trust. + +For a `pipx` installation of MVT, inject the package into MVT's environment: + +```bash +pipx inject mvt mvt-plugin-example-org +``` + +Module packages that need their own settings, such as an API key, should store +them in a namespaced [plugin configuration file](plugin_configuration.md) +rather than in MVT's own `config.yaml`. + +### Naming module packages + +Name module packages `mvt-plugin-` (import package `mvt_plugin_`), +and include the name of the publishing organization or author so packages from +different groups do not collide: for example, an organisation's custom modules +would be distributed as `mvt-plugin-example-org` with the import package +`mvt_plugin_example_org`. + +The prefix makes module packages easy to find on PyPI and keeps their import +names from clashing with unrelated Python packages. It is a convention, not a +technical requirement: modules load through the `mvt.modules` entry point +regardless of what the package is called, and existing packages with other +names keep working. Note that the prefix is also not a mark of authenticity — +anyone can publish a package with any available name, so vet a module package +and its publisher before installing it, whatever it is called. + +### Module logging + +Modules log through `self.log`, and MVT names the logger for where the module +came from. MVT's own modules log under their dotted path (for example +`mvt.ios.modules.mixed.whatsapp`). Everything external is namespaced under +`mvt.ext` to keep it visually distinct from built-in modules and isolated from +MVT's internal logger tree: + +- Installed packages log under `mvt.ext.`, with the `mvt_plugin_` + prefix stripped: modules in `mvt_plugin_example_org` log as + `mvt.ext.example_org.*`. +- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as + `mvt.ext.`. + +Outside a module class, for example in a custom command line handler, log +through `get_plugin_logger(__name__)`. It returns a logger in the same +namespace. + +## Developing modules locally + +While a module is being written, load it from its file. `--load-module` takes a +Python file, or a folder of them, on every module-running command, and can be +repeated: + +```bash +mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup +``` + +For a folder, MVT loads its non-hidden top-level `*.py` files in sorted order +and skips `__init__.py`. `MVT_CUSTOM_MODULES` names a folder to load on every +module-running command, before any `--load-module` path: + +```bash +MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip +``` + +Files loaded this way follow the same rules as packaged modules. +`--list-modules` reports them with the SHA-256 hash of the file in place of a +version. An editable install of the package (`pip install -e .`) also works: +the modules load through the entry point, and `mvt plugins list` shows the +package with the `local` origin. + +Loading by path is for development. Move a module into a package once it +works. + +## Auditing loaded modules + +Because installed module packages load automatically, MVT records where every +module came from: + +- `--list-modules` groups the available modules by source: MVT itself + (with its version), each installed package (with its version and, when + installed directly from a repository, the commit), and each file loaded + with `--load-module` or `MVT_CUSTOM_MODULES` (with the SHA-256 hash of the + file). +- When a command runs with an `--output` folder, the `command.log` file + records one line per module source with the source's version or hash and + the list of modules loaded from it. +- `mvt plugins list` lists the installed packages, where each of them + was installed from and how many modules it contributes, see + [Managing Plugins](plugins.md). + +## Profiling + +Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be +take to avoid inefficient code paths as we add new modules. + +MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable. + +```bash +MVT_PROFILE=1 dev/mvt-ios check-backup test_backup +``` + +Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT. diff --git a/docs/development/plugin_configuration.md b/docs/development/plugin_configuration.md new file mode 100644 index 0000000..62d35a7 --- /dev/null +++ b/docs/development/plugin_configuration.md @@ -0,0 +1,157 @@ +# Plugin Configuration + +Plugin packages that add [custom CLI commands](custom_commands.md) or +[modules](index.md#installed-module-packages) often need to store their +own settings, such as an API key, a server URL or the timestamp of the last +synchronization. MVT provides a namespaced settings base class so each plugin +keeps its configuration in its own file, and a data folder for anything else a +plugin needs to keep on disk. + +!!! warning + + Do not write plugin settings to MVT's own `config.yaml`. MVT rewrites that + file with the settings it knows about every time it starts, so any other + section is deleted. + +## Where Settings Are Stored + +Each plugin gets one YAML file in a `plugins` folder next to MVT's own +configuration: + +``` +~/.config/mvt/plugins/.yaml +``` + +The exact parent folder follows the platform convention used for MVT's +`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use +`plugin_config_path()` from `mvt.plugin` instead of building the path by hand. + +Plugin names must be lowercase and may only contain letters, digits and dashes, +matching the `mvt-plugin-` package naming convention. MVT creates the +`plugins` folder with `0700` permissions and writes the settings files with +`0600` permissions, because they commonly hold credentials. Files are written +through a temporary file and moved into place, so an interrupted save never +leaves a partially written settings file behind. + +## Plugin Data Folder + +Everything else a plugin keeps on disk, such as a cache, a downloaded artifact +or synchronization state, belongs in the folder returned by the `data_folder()` +class method of the plugin's settings class, or by `plugin_data_folder()` from +`mvt.plugin`, called with the plugin name if the plugin has no settings class: + +``` +~/.local/share/mvt/plugin-data// # Linux +~/Library/Application Support/mvt/plugin-data// # macOS +``` + +The folder sits beside MVT's own data, such as the downloaded indicators. It is +created if it is missing, with `0700` permissions. Asking for it again returns +the same path and leaves the contents alone, so a plugin can ask for it every +time it needs the folder. `ExamplePluginSettings` below is the settings class +defined in the next section: + +```python +import os + + +def cache_path() -> str: + folder = ExamplePluginSettings.data_folder() + return os.path.join(folder, "virustotal_lookups_cache.json") +``` + +A plugin which has no settings class calls +`plugin_data_folder("example-plugin")` instead. + +Do not fall back on a path of your own such as `~/.cache/example-plugin`: it +is a Linux-only convention, and MVT will not create it for you. + +## Defining Plugin Settings + +Subclass `MVTPluginSettings`, set `plugin_name` and declare typed fields with +defaults: + +```python +from typing import Optional + +from mvt.plugin import MVTPluginSettings + + +class ExamplePluginSettings(MVTPluginSettings): + plugin_name = "example-plugin" + + API_KEY: Optional[str] = None + MAX_RESULTS: int = 25 + LAST_SYNC: Optional[str] = None +``` + +`load()` returns the current settings and `save()` writes them back: + +```python +from datetime import datetime, timezone + +import click + +from mvt.plugin import plugin_env_prefix + + +def sync(): + settings = ExamplePluginSettings.load() + if not settings.API_KEY: + prefix = plugin_env_prefix(settings.plugin_name) + raise click.ClickException( + f"No API key configured. Set {prefix}API_KEY or " + "run 'example-plugin configure'." + ) + + settings.LAST_SYNC = datetime.now(timezone.utc).isoformat() + settings.save() +``` + +The message builds the variable name with `plugin_env_prefix()`, see +[Environment Variables](#environment-variables). + +A missing settings file is not an error: the plugin then runs on the field +defaults and on whatever the environment provides. `save()` only persists the +values that differ from the defaults, and it never touches MVT's `config.yaml`. +A settings file that cannot be parsed, or that does not hold a mapping of +setting names to values, raises a `PluginConfigLoadError` naming the file. + +Every subclass that sets its own `plugin_name` gets its own file and its own +environment namespace. A subclass that does not redefine `plugin_name` inherits +it, and therefore shares the file and the environment variables of its parent +class. + +## Environment Variables + +Every field can also be set with an environment variable. The prefix is +`MVT_PLUGIN_`, followed by the plugin name upper-cased with dashes replaced by +underscores, followed by the field name. For the example above: + +```bash +export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=... +export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50 +``` + +Do not repeat the plugin name in a field name: a plugin named `example-scanner` +with a `SCANNER_API_KEY` field asks the user for +`MVT_PLUGIN_EXAMPLE_SCANNER_SCANNER_API_KEY`. Name the field `API_KEY`. + +Settings resolve in this order, from highest to lowest priority: + +1. Arguments passed to the settings class directly, such as + `ExamplePluginSettings(API_KEY="...")` +2. Environment variables +3. The plugin's YAML file +4. The field defaults declared on the settings class + +!!! tip + + On shared or multi-user machines, prefer passing API keys through + environment variables rather than saving them to the plugin file. `save()` + skips every value that the environment currently supplies, so a credential + provided that way is not copied into the settings file when a plugin saves + an unrelated setting. + +Unknown keys in a plugin's YAML file are ignored, so a settings file written by +a newer version of a plugin does not break an older one. diff --git a/docs/development/plugins.md b/docs/development/plugins.md new file mode 100644 index 0000000..0085975 --- /dev/null +++ b/docs/development/plugins.md @@ -0,0 +1,116 @@ +# Managing Plugins + +Plugin packages extend MVT with additional +[forensic modules](index.md#custom-modules) and +[CLI commands](custom_commands.md). Because installed packages load +automatically, `mvt plugins` audits what is installed and checks whether +updates are available. The command lives on `mvt` only, although the packages +it lists extend `mvt-ios` and `mvt-android` too. + +## List Installed Plugins + +```bash +mvt plugins list +``` + +``` + Installed MVT plugins +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓ +┃ Name ┃ Version ┃ Origin ┃ Modules ┃ Commands ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩ +│ mvt-plugin-example │ 1.2.0 │ pypi │ 4 │ summarize │ +│ mvt-plugin-research │ 0.1.0 │ git+3f9a1c7d │ 2 │ - │ +│ mvt-plugin-local │ 0.0.1 │ local │ 1 │ triage │ +└─────────────────────┴─────────┴──────────────┴─────────┴───────────┘ +``` + +The origin records where each package was installed from: `pypi` for a package +installed from a package index, `git+` for a package installed directly +from a repository, and `local` for a package installed from a local folder or +archive rather than from an index, including an editable development install. +The last two columns show how many forensic modules the package contributes and +which CLI commands it adds. + +A plugin whose modules cannot be imported is listed with `error` in the +`Modules` column rather than breaking the listing. + +## Check for Updates + +```bash +mvt plugins check-updates +``` + +``` +Plugin updates available: + mvt-plugin-example 1.2.0 → 1.3.0 + Upgrade with: pip install -U mvt-plugin-example + +MVT does not install plugin updates. Run the command above when you decide to +upgrade. +``` + +Packages installed from a package index are compared against the latest release +published for them. A package which was never published, for example a plugin +distributed only within an organization, is skipped silently. + +!!! note + + Packages shown with the `pypi` origin are compared against + [PyPI](https://pypi.org), whichever index they were installed from. A + plugin installed from a private index under a name which also exists on + PyPI is therefore compared against the unrelated public package of that + name. Give plugins published to a private index a name which is not taken + on PyPI, and treat an unexpected update suggestion as a reason to check + where the package would come from. + +!!! warning + + MVT never installs or upgrades a plugin itself, it only prints the command + which does. Upgrading a plugin in the middle of an investigation changes + the modules producing the results, and a plugin runs as trusted code inside + the MVT process, so pulling in a new version is a decision for the analyst + to make deliberately and not a side effect of running a check. + +## Automatic Update Checks + +MVT also reports available plugin updates in the banner printed when a command +starts: + +``` + MVT - Mobile Verification Toolkit + + https://mvt.re + Version: 2026.7.29 + + Plugin updates available: + mvt-plugin-example 1.2.0 → 1.3.0 (pip install -U mvt-plugin-example) +``` + +This check runs at most once every 12 hours. In between checks MVT prints the +findings of the latest check without contacting anything, so a plugin update +stays visible without a lookup on every command. The +`mvt plugins check-updates` command checks immediately, regardless of when the +last check happened. + +The automatic check is skipped when the `--disable-update-check` option is +used, when `NETWORK_ACCESS_ALLOWED` is disabled in the MVT configuration, and +when no plugins are installed. + +## Plugins Installed From a Repository + +A plugin installed with `pip install "mvt-plugin-example @ git+"` is +checked by asking the remote repository which commit the installed revision +points at now. MVT runs git and ssh in batch mode, so a repository which needs +credentials MVT does not already have fails the check instead of prompting for +them. The check is skipped silently when git is not available, when the +repository cannot be reached, and when access to it is denied. + +How the plugin was installed decides what an update means: + +- A plugin installed from a branch is reported as outdated when the branch has + moved past the installed commit. +- A plugin installed from a specific commit or a tag is pinned. It is never + reported as outdated, however far the branch it came from moves on. + +Pinning a plugin to a commit or a tag is therefore the way to keep the modules +used across an investigation stable. diff --git a/docs/docker.md b/docs/docker.md index d8f4ff8..453c245 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -31,4 +31,18 @@ Test if the image was created successfully: docker run -it mvt ``` -If a prompt is spawned successfully, you can close it with `exit`. \ No newline at end of file +If a prompt is spawned successfully, you can close it with `exit`. + +## Access an iOS device from Docker + +On the Linux host, install and start [usbmuxd](https://github.com/libimobiledevice/usbmuxd), then connect and unlock the iOS device. The daemon exposes the device through the `/var/run/usbmuxd` socket. + +Bind that socket into the container to let MVT communicate with the device: + +```bash +docker run -it \ + --mount type=bind,source=/var/run/usbmuxd,target=/var/run/usbmuxd \ + ghcr.io/mvt-project/mvt +``` + +If you built the image from source, replace `ghcr.io/mvt-project/mvt` with `mvt`. diff --git a/docs/index.md b/docs/index.md index fada316..8cd733e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forens It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors. -In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results. +In this documentation you will find instructions on how to install and run the `mvt-ios`, `mvt-android` and `mvt` commands, and guidance on how to interpret the extracted results. ## Resources diff --git a/docs/install.md b/docs/install.md index c08c75d..c024338 100644 --- a/docs/install.md +++ b/docs/install.md @@ -64,7 +64,7 @@ It is recommended to try installing and running MVT from [Windows Subsystem Linu pipx install mvt ``` -You now should have the `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window. +You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window. ### Installing from PyPI directly into a virtual environment You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool: @@ -84,7 +84,7 @@ source env/bin/activate pip install mvt ``` -The `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active. +The `mvt`, `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active. ### Installing from git source with pipx If you want to have the latest features in development, you can install MVT directly from the source code in git. @@ -93,7 +93,7 @@ If you want to have the latest features in development, you can install MVT dire pipx install --force git+https://github.com/mvt-project/mvt.git ``` -You now should have the `mvt-ios` and `mvt-android` utilities installed. +You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. **Notes:** 1. The `--force` flag is necessary to force the reinstallation of the package. diff --git a/docs/iocs.md b/docs/iocs.md index c7586de..5f50eff 100644 --- a/docs/iocs.md +++ b/docs/iocs.md @@ -71,7 +71,7 @@ So far MVT implements only a subset of [STIX2 specifications](https://docs.oasis - [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2). - We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns. -You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT. +You can automatically download the latest public indicator files with the command `mvt download-iocs`. The per-platform forms `mvt-ios download-iocs` and `mvt-android download-iocs` do the same thing. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT. Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs. diff --git a/docs/ios/records.md b/docs/ios/records.md index 016c861..b34183a 100644 --- a/docs/ios/records.md +++ b/docs/ios/records.md @@ -417,7 +417,21 @@ If indicators are provided through the command-line, they are checked against th Backup: :material-check: Full filesystem dump: :material-check: -This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*. +This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*, along with one record per chat session (marked with `"record_type": "chat_session"`) containing the first and last interaction dates of each conversation. Chat sessions produce `chat_first_message` and `chat_last_message` timeline events, and group chats additionally produce a `group_created` event. A chat session's last-message date can postdate its newest stored message when the most recent messages in the chat were deleted. + +Recent WhatsApp versions key 1:1 chat sessions by an opaque LID identifier rather than the contact's phone number. The module resolves these using the `ZWAPHONENUMBERLIDPAIR` table from the *LID.sqlite* database in the same app group (or from *ChatStorage.sqlite* itself in versions that store it there), populating `partner_resolved_phone_number` on chat session records and using the phone number in timeline events. Each LID-phone number pair is also extracted as a record (`"record_type": "lid_phone_number_pair"`) and produces a `lid_pair_recorded` timeline event marking when WhatsApp learned the association. If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*. +--- + +### `whatsapp_contacts.json` + +!!! info "Availability" + Backup: :material-check: + Full filesystem dump: :material-check: + +This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extracts WhatsApp contact records from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ContactsV2.sqlite*, including each contact's phone number, WhatsApp and LID identifiers, and the per-contact disappearing messages timer, which is not recorded in *ChatStorage.sqlite*. Each timestamp stored on a contact record produces a timeline event: `disappearing_mode_set` (when the disappearing messages timer was last changed), `about_changed` (when the contact last changed their "about" text), `about_expiration` (when a timed "about" is scheduled to expire) and `contact_last_updated` (when the contact record was last updated). + +This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup. + diff --git a/docs/ios/sysdiagnose.md b/docs/ios/sysdiagnose.md index 6488bc2..d0d84fb 100644 --- a/docs/ios/sysdiagnose.md +++ b/docs/ios/sysdiagnose.md @@ -1,30 +1,32 @@ # Check an iOS Sysdiagnose `mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by -custom MVT modules. MVT does not include built-in sysdiagnose modules. You must -load at least one custom module that explicitly supports this command. +custom MVT modules. MVT does not include built-in sysdiagnose modules. The +command runs the modules of the installed +[plugin packages](../development/index.md#installed-module-packages) which +declare support for it. Install at least one such package first. The command accepts either an extracted sysdiagnose directory or the original gzip-compressed tar archive. ```bash -mvt-ios check-sysdiagnose \ - --load-module ./sysdiagnose_modules.py \ - --output ./results \ +mvt-ios check-sysdiagnose --output ./results \ ./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz ``` Use `--hashes` to include hashes for analyzed files in `info.json`, and -`--list-modules` to display the eligible custom modules without running them. +`--list-modules` to display the eligible modules without running them. ## Writing a custom module -Extend `SysdiagnoseExtraction` to access the archive contents consistently for -both directory and tar inputs. Each module must declare the command explicitly -in `supported_commands`. +Extend `SysdiagnoseExtraction` from `mvt.plugin`, see +[Writing a module](../development/index.md#writing-a-module). The module reads +the archive the same way whether MVT was given a folder or a tar archive. It +declares the command in `supported_commands`. While writing one, +[load it from its file](../development/index.md#developing-modules-locally). ```python -from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction +from mvt.plugin import SysdiagnoseExtraction class ExampleSysdiagnoseModule(SysdiagnoseExtraction): @@ -44,7 +46,9 @@ class ExampleSysdiagnoseModule(SysdiagnoseExtraction): return None ``` -The base class provides `from_sysdiagnose_folder()` and -`from_sysdiagnose_tar()` setup hooks, as well as protected file lookup, file -reading, and timezone extraction helpers. IPS crash-report metadata is exposed -on `ips_files`. +MVT extracts a tar archive first. It calls `from_sysdiagnose_folder()` on each +module before `run()`. `ips_files` lists the IPS crash reports. + +`_get_files_by_pattern()` and `_get_file_content()` are internal helpers of the +base class. Use them to read the archive. Their names and signatures can change +between releases. See `src/mvt/ios/modules/sysdiagnose/base.py`. diff --git a/mkdocs.yml b/mkdocs.yml index 66f3471..66def17 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,7 +31,6 @@ nav: - Introduction: "introduction.md" - Installation: "install.md" - Command Completion: "command_completion.md" - - Custom CLI Commands: "custom_commands.md" - Using Docker: "docker.md" - MVT for iOS: - iOS Forensic Methodology: "ios/methodology.md" @@ -50,5 +49,9 @@ nav: - Check an Android Backup (SMS messages): "android/backup.md" - Check Android Intrusion Logs: "android/intrusion_logs.md" - Indicators of Compromise: "iocs.md" - - Development: "development.md" + - Development: + - Development Instructions: "development/index.md" + - Custom CLI Commands: "development/custom_commands.md" + - Plugin Configuration: "development/plugin_configuration.md" + - Managing Plugins: "development/plugins.md" - License: "license.md" diff --git a/pyproject.toml b/pyproject.toml index 5548e40..9f423d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,9 @@ homepage = "https://docs.mvt.re/en/latest/" repository = "https://github.com/mvt-project/mvt" [project.scripts] -mvt-ios = "mvt.ios:cli" -mvt-android = "mvt.android:cli" +mvt = "mvt.cli:main" +mvt-ios = "mvt.ios:main" +mvt-android = "mvt.android:main" [dependency-groups] dev = [ @@ -72,7 +73,7 @@ docs = [ ] [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=61.0", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" [tool.coverage.run] @@ -121,5 +122,6 @@ where = ["src"] [tool.setuptools.package-data] mvt = ["ios/data/*.json"] -[tool.setuptools.dynamic] -version = { attr = "mvt.common.version.MVT_VERSION" } +[tool.setuptools_scm] +# The version is the latest v* tag; ignore the archive/* tags. +git_describe_command = "git describe --dirty --tags --long --match 'v*'" diff --git a/src/mvt/android/__init__.py b/src/mvt/android/__init__.py index 2c05f56..6616bcb 100644 --- a/src/mvt/android/__init__.py +++ b/src/mvt/android/__init__.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from .cli import cli +from .cli import cli, main diff --git a/src/mvt/android/artifacts/dumpsys_accessibility.py b/src/mvt/android/artifacts/dumpsys_accessibility.py index 3d8fa6c..da46043 100644 --- a/src/mvt/android/artifacts/dumpsys_accessibility.py +++ b/src/mvt/android/artifacts/dumpsys_accessibility.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ import re +from typing import Any from .artifact import AndroidArtifact @@ -20,10 +21,10 @@ class DumpsysAccessibilityArtifact(AndroidArtifact): continue self.alertstore.medium( - f'Found accessibility service: "{result["service"]}"', + f'Found accessibility service: "{result["component"]}"', "", result, - ) + ) def parse(self, content: str) -> None: """ @@ -33,41 +34,69 @@ class DumpsysAccessibilityArtifact(AndroidArtifact): :param content: content of the accessibility section (string) """ - # "Old" syntax - in_services = False + self.results: list[dict[str, Any]] = [] + services: dict[tuple[int | None, str], dict] = {} + user_id: int | None = None + state: str | None = None + for line in content.splitlines(): - if line.strip().startswith("installed services:"): - in_services = True - continue + user_match = re.search(r"attributes:\{id=(\d+)", line) + if user_match: + user_id = int(user_match.group(1)) - if not in_services: - continue - - if line.strip() == "}": - # At end of installed services - break - - service = line.split(":")[1].strip() - - self.results.append( - { - "package_name": service.split("/")[0], - "service": service, - } + stripped = line.strip() + state_match = re.match( + r"(?i)(installed|enabled|binding|bound|crashed) services\s*:\s*\{(.*)", + stripped, ) - - # "New" syntax - AOSP >= 14 (?) - # Looks like: - # Enabled services:{{com.azure.authenticator/com.microsoft.brooklyn.module.accessibility.BrooklynAccessibilityService}, {com.agilebits.onepassword/com.agilebits.onepassword.filling.accessibility.FillingAccessibilityService}} - - for line in content.splitlines(): - if line.strip().startswith("Enabled services:"): - matches = re.finditer(r"{([^{]+?)}", line) - - for match in matches: - # Each match is in format: / - package_name, _, service = match.group(1).partition("/") - - self.results.append( - {"package_name": package_name, "service": service} + if state_match: + state = state_match.group(1).lower() + inline = state_match.group(2) + for component in re.findall( + r"\{?([\w.$-]+/[\w.$-]+)(?:\s+\(A11yTool\))?\}?", inline + ): + service = services.setdefault( + (user_id, component), self._new_service(component, user_id) ) + service[self._state_field(state)] = True + service["accessibility_tool"] = "(A11yTool)" in inline + continue + + if not state: + continue + if stripped == "}" or stripped.startswith("AccessibilityInputFilter"): + state = None + continue + component_match = re.search( + r"(?:\d+\s*:\s*)?([\w.$-]+/[\w.$-]+)(?:\s+\(A11yTool\))?", + stripped, + ) + if component_match: + component = component_match.group(1) + service = services.setdefault( + (user_id, component), self._new_service(component, user_id) + ) + service[self._state_field(state)] = True + service["accessibility_tool"] = "(A11yTool)" in stripped + + self.results.extend(services.values()) + + @staticmethod + def _state_field(state: str) -> str: + return {"binding": "binding", "bound": "bound"}.get(state, state) + + @staticmethod + def _new_service(component: str, user_id: int | None) -> dict: + package_name, service_name = component.split("/", 1) + return { + "user_id": user_id, + "component": component, + "package_name": package_name, + "service_name": service_name, + "installed": False, + "enabled": False, + "binding": False, + "bound": False, + "crashed": False, + "accessibility_tool": False, + } diff --git a/src/mvt/android/artifacts/dumpsys_adb.py b/src/mvt/android/artifacts/dumpsys_adb.py index e5438c9..cbd80e2 100644 --- a/src/mvt/android/artifacts/dumpsys_adb.py +++ b/src/mvt/android/artifacts/dumpsys_adb.py @@ -6,6 +6,7 @@ import base64 import binascii import hashlib +import re from .artifact import AndroidArtifact @@ -98,6 +99,34 @@ class DumpsysADBArtifact(AndroidArtifact): return keystore + def parse_binary_xml(self, data: bytes) -> list[dict]: + """Recover ADB key records from Android binary XML (ABX). + + Some dumpstate implementations embed ABX in a text stream and replace + binary token bytes. The public key remains intact, while unavailable + numeric metadata is represented as ``None`` rather than corrupt text. + """ + keystore = [] + seen = set() + for match in re.finditer( + rb"(? dict: if b" " in user_key: @@ -118,7 +147,7 @@ class DumpsysADBArtifact(AndroidArtifact): return { "user": user.decode("utf-8"), "fingerprint": key_fingerprint_colon, - "key": key_base64, + "key": key_base64.decode("ascii", errors="replace"), } def check_indicators(self) -> None: @@ -175,12 +204,23 @@ class DumpsysADBArtifact(AndroidArtifact): # Keystore is in XML format on some devices and we need to parse it if keystore_data and keystore_data.startswith(b" None: - # self.results: List[Dict[str, Any]] = [] - perm: dict[str, Any] = {} - package: dict[str, Any] = {} - entry: dict[str, Any] = {} - uid = None + self.results: list[dict[str, Any]] = [] + permission: dict[str, Any] | None = None + package: dict[str, Any] | None = None + uid: str | None = None + uid_details: dict[str, Any] = {} + attribution: str | None = None in_packages = False + def finish_permission() -> None: + nonlocal permission + if package is not None and permission is not None: + package["permissions"].append(permission) + permission = None + + def finish_package() -> None: + nonlocal package + finish_permission() + if package is not None: + self.results.append(package) + package = None + for line in output.splitlines(): - if line.startswith(" Uid 0:"): + uid_match = re.match(r"^ Uid ([^:]+):$", line) + if uid_match: in_packages = True + finish_package() + uid = uid_match.group(1) + uid_details = { + "uid_state": None, + "capability": None, + "app_widget_visible": None, + "default_modes": {}, + } + continue if not in_packages: continue - if line.startswith(" Uid "): - uid = line[6:-1] - if entry: - perm["entries"].append(entry) - entry = {} - if package: - if perm: - package["permissions"].append(perm) + uid_property = re.match( + r"^ (state|capability|appWidgetVisible)=(.*)$", line + ) + if uid_property: + key = { + "state": "uid_state", + "appWidgetVisible": "app_widget_visible", + }.get(uid_property.group(1), uid_property.group(1)) + value: Any = uid_property.group(2) + if value in ("true", "false"): + value = value == "true" + uid_details[key] = value + continue - perm = {} - self.results.append(package) - package = {} + default_mode = re.match(r"^ ([A-Z0-9_]+): mode=([^\s]+)", line) + if default_mode and package is None: + uid_details["default_modes"][default_mode.group(1)] = ( + default_mode.group(2) + ) continue if line.startswith(" Package "): - if entry: - perm["entries"].append(entry) - entry = {} - - if package: - if perm: - package["permissions"].append(perm) - - perm = {} - self.results.append(package) - + finish_package() package = { "package_name": line[12:-1], "permissions": [], "uid": uid, + **uid_details, } continue - if package and line.startswith(" ") and line[6] != " ": - if entry: - perm["entries"].append(entry) - entry = {} - if perm: - package["permissions"].append(perm) - perm = {} - - perm["name"] = line.split()[0] - perm["entries"] = [] - if len(line.split()) > 1: - perm["access"] = line.split()[1][1:-2] - + operation_match = re.match( + r"^ ([A-Z0-9_]+)(?: \(([^)]+)\))?:\s*$", line + ) + if package is not None and operation_match: + finish_permission() + permission = { + "name": operation_match.group(1), + "mode": operation_match.group(2), + "entries": [], + } + attribution = None continue - if line.startswith(" "): - # Permission entry like: - # Reject: [fg-s]2021-05-19 22:02:52.054 (-314d1h25m2s33ms) - access_type = line.split(":")[0].strip() - if access_type not in ["Access", "Reject"]: - # Skipping invalid access type. Some entries are not in the format we expect - continue + attribution_match = re.match(r"^\s{8,}([^=]+)=\[$", line) + if attribution_match: + attribution = attribution_match.group(1).strip() + continue + if line.strip() == "]": + attribution = None + continue - if entry: - perm["entries"].append(entry) - entry = {} - - entry["access"] = access_type - entry["type"] = line[line.find("[") + 1 : line.find("]")] - - try: - entry["timestamp"] = convert_datetime_to_iso( - datetime.strptime( - line[line.find("]") + 1 : line.find("(")].strip(), - "%Y-%m-%d %H:%M:%S.%f", - ) + if permission is None: + continue + event_match = re.match( + r"^\s*(Access|Reject):\s*\[([^]]+)\]\s*" + r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)\s*" + r"(\([^)]*\))?(?:\s+duration=([^\s]+))?", + line, + ) + running_match = re.match( + r"^\s*Running start at:\s*(\S+(?: \S+)?)", + line, + ) + if event_match: + entry = { + "event": event_match.group(1).lower(), + "access": event_match.group(1), + "uid_state": event_match.group(2), + "timestamp": convert_datetime_to_iso( + datetime.strptime(event_match.group(3), "%Y-%m-%d %H:%M:%S.%f") + ), + "relative_time": event_match.group(4), + "duration": event_match.group(5), + "attribution": attribution, + } + permission["entries"].append(entry) + elif running_match: + raw_start = running_match.group(1) + timestamp = None + if re.fullmatch(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+", raw_start): + timestamp = convert_datetime_to_iso( + datetime.strptime(raw_start, "%Y-%m-%d %H:%M:%S.%f") ) - except ValueError: - # Invalid date format - pass + permission["entries"].append( + { + "event": "running", + "access": "Running", + "uid_state": None, + "timestamp": timestamp, + "relative_time": raw_start + if raw_start.startswith("+") + else None, + "duration": None, + "attribution": attribution, + } + ) - if line.strip() == "": - break - - if entry: - perm["entries"].append(entry) - if perm: - package["permissions"].append(perm) - if package: - self.results.append(package) + finish_package() diff --git a/src/mvt/android/artifacts/dumpsys_battery_daily.py b/src/mvt/android/artifacts/dumpsys_battery_daily.py index c0f2c5e..2ccb6b7 100644 --- a/src/mvt/android/artifacts/dumpsys_battery_daily.py +++ b/src/mvt/android/artifacts/dumpsys_battery_daily.py @@ -18,18 +18,18 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: action = record.get("action", "update") package_name = record["package_name"] - vers = record["vers"] + vers = record["version_code"] - if vers == "0": + if vers == 0: data = f"Recorded uninstall of package {package_name} (vers 0)" elif action == "downgrade": - prev_vers = record.get("previous_vers", "unknown") + prev_vers = record.get("previous_version_code", "unknown") data = f"Recorded downgrade of package {package_name} from vers {prev_vers} to vers {vers}" else: data = f"Recorded update of package {package_name} with vers {vers}" return { - "timestamp": record["from"], + "timestamp": record["period_start"], "module": self.__class__.__name__, "event": "battery_daily", "data": data, @@ -48,6 +48,7 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): continue def parse(self, output: str) -> None: + self.results = [] daily = None daily_updates: list[dict[str, Any]] = [] records: list[dict[str, Any]] = [] @@ -59,7 +60,10 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): timeframe = line[13:].strip() date_from, date_to = timeframe.strip(":").split(" to ", 1) - daily = {"from": date_from[0:10], "to": date_to[0:10]} + daily = { + "period_start": self._format_daily_timestamp(date_from), + "period_end": self._format_daily_timestamp(date_to), + } continue if not daily: @@ -70,21 +74,30 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): line = line.strip().replace("Update ", "") package_name, vers = line.split(" ", 1) - vers_nr = vers.split("=", 1)[1] + vers_raw = vers.split("=", 1)[1] + try: + version_code: int | str = int(vers_raw) + except ValueError: + version_code = vers_raw already_seen = False for update in daily_updates: - if package_name == update["package_name"] and vers_nr == update["vers"]: + if ( + package_name == update["package_name"] + and version_code == update["version_code"] + ): + update["occurrences"] += 1 already_seen = True break if not already_seen: update_record: dict[str, Any] = { "action": "update", - "from": daily["from"], - "to": daily["to"], + "period_start": daily["period_start"], + "period_end": daily["period_end"], "package_name": package_name, - "vers": vers_nr, + "version_code": version_code, + "occurrences": 1, } daily_updates.append(update_record) @@ -95,26 +108,31 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): self._detect_uninstalls_and_downgrades(records) self.results.extend(records) - def _detect_uninstalls_and_downgrades( - self, records: list[dict[str, Any]] - ) -> None: + @staticmethod + def _format_daily_timestamp(value: str) -> str: + if len(value) >= 19 and value[10] == "-": + return f"{value[:10]} {value[11:].replace('-', ':')}" + return value + + def _detect_uninstalls_and_downgrades(self, records: list[dict[str, Any]]) -> None: package_versions: dict[str, int] = {} for record in sorted( records, key=lambda record: ( - record["from"], - record["to"], + record["period_start"], + record["period_end"], record["package_name"], ), ): package_name = record["package_name"] - vers_nr = record["vers"] + vers_nr = record["version_code"] - if vers_nr == "0": + if vers_nr == 0: + record["action"] = "uninstall" self.alertstore.medium( f"Detected uninstall of package {package_name} (vers 0)", - record["from"], + record["period_start"], record, ) package_versions.pop(package_name, None) @@ -128,11 +146,11 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact): previous_vers = package_versions.get(package_name) if previous_vers is not None and current_vers < previous_vers: record["action"] = "downgrade" - record["previous_vers"] = str(previous_vers) + record["previous_version_code"] = previous_vers self.alertstore.medium( f"Detected downgrade of package {package_name} " f"from vers {previous_vers} to vers {current_vers}", - record["from"], + record["period_start"], record, ) diff --git a/src/mvt/android/artifacts/dumpsys_battery_history.py b/src/mvt/android/artifacts/dumpsys_battery_history.py index 77d9d3c..6cb36f1 100644 --- a/src/mvt/android/artifacts/dumpsys_battery_history.py +++ b/src/mvt/android/artifacts/dumpsys_battery_history.py @@ -1,96 +1,177 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021-2023 The MVT Authors. +# Copyright (c) 2021-2026 The MVT Authors. # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import datetime +import re + +from mvt.common.utils import convert_datetime_to_iso + from .artifact import AndroidArtifact class DumpsysBatteryHistoryArtifact(AndroidArtifact): - """ - Parser for dumpsys dattery history events. - """ + """Parser for package-related events in dumpsys batterystats history.""" def check_indicators(self) -> None: if not self.indicators: return - for result in self.results: - ioc_match = self.indicators.check_app_id(result["package_name"]) + package_name = result.get("package_name") + if not package_name: + continue + ioc_match = self.indicators.check_app_id(package_name) if ioc_match: self.alertstore.critical( ioc_match.message, "", result, matched_indicator=ioc_match.ioc ) - continue + + @staticmethod + def _parse_wall_time(value: str) -> datetime.datetime | None: + if re.fullmatch(r"\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+", value): + value = f"1900-{value}" + for date_format in ( + "%Y-%m-%d-%H-%M-%S-%f", + "%Y-%m-%d-%H-%M-%S", + "%Y-%m-%d %H:%M:%S.%f", + ): + try: + return datetime.datetime.strptime(value, date_format) + except ValueError: + pass + return None + + @staticmethod + def _elapsed_seconds(value: str) -> float | None: + if not value.startswith("+"): + return None + units = {"d": 86400, "h": 3600, "m": 60, "s": 1, "ms": 0.001} + total = 0.0 + for number, unit in re.findall(r"(\d+)(ms|d|h|m|s)", value): + total += int(number) * units[unit] + return total + + @staticmethod + def _package_from_name(name: str) -> str | None: + clean = name.removeprefix("*walarm*:").removeprefix("*alarm*:") + slash_parts = clean.split("/") + if len(slash_parts) > 1: + first = slash_parts[0].lstrip("@") + if first.startswith(("com.", "org.", "net.")): + return first + for part in reversed(slash_parts[1:]): + candidate = part.lstrip("@").split(":", 1)[0] + if candidate.startswith(("com.", "org.", "net.")): + return candidate + parts = clean.split(".") + package_parts = [] + for part in parts: + if part and part[0].islower(): + package_parts.append(part) + else: + break + return ".".join(package_parts) if len(package_parts) >= 2 else None + + @staticmethod + def _normalize_service(service: str) -> str: + # WorkManager decorates jobs with one or more scheduler prefixes. + if "@" in service: + candidates = [part for part in service.split("@") if "/" in part] + if candidates: + return candidates[-1] + return service def parse(self, data: str) -> None: + self.results: list[dict[str, str | None]] = [] + anchor_time: datetime.datetime | None = None + anchor_elapsed = 0.0 + has_history_heading = any( + line.startswith("Battery History") for line in data.splitlines() + ) + in_history = not has_history_heading + for line in data.splitlines(): - if line.startswith("Battery History "): + stripped = line.strip() + if line.startswith("Battery History"): + if in_history: + break + in_history = True continue - - if line.strip() == "": + if not in_history: + continue + if has_history_heading and not stripped: break - - time_parts = line.strip().split() - time_elapsed = time_parts[0] - if ( - len(time_parts) > 1 - and len(time_parts[0]) == 5 - and time_parts[0][2] == "-" - and ":" in time_parts[1] - ): - time_elapsed = " ".join(time_parts[:2]) - - event = "" - if line.find("+job") > 0: - event = "start_job" - payload = line.split("+job=", 1)[1] - uid, separator, service = payload.partition(":") - if not separator: - continue - service = service.strip().strip('"') - package_name = service.split("/")[0] - elif line.find("-job") > 0: - event = "end_job" - payload = line.split("-job=", 1)[1] - uid, separator, service = payload.partition(":") - if not separator: - continue - service = service.strip().strip('"') - package_name = service.split("/")[0] - elif line.find("+running +wake_lock=") > 0: - payload = line.split("+running +wake_lock=", 1)[1] - uid, separator, _ = payload.partition(":") - if not separator: - continue - event = "wake" - service = ( - line[line.find("*walarm*:") + 9 :].split(" ")[0].strip('"').strip() - ) - if service == "" or "/" not in service: - continue - - package_name = service.split("/")[0] - elif (line.find("+top=") > 0) or (line.find("-top") > 0): - if line.find("+top=") > 0: - event = "start_top" - top_pos = line.find("+top=") - else: - event = "end_top" - top_pos = line.find("-top=") - colon_pos = top_pos + line[top_pos:].find(":") - uid = line[top_pos + 5 : colon_pos] - service = "" - package_name = line[colon_pos + 1 :].strip('"') - else: + reset_match = re.search(r"(?:RESET:)?TIME:\s*(\S+)", stripped) + if reset_match: + parsed_time = self._parse_wall_time(reset_match.group(1)) + if parsed_time is not None: + elapsed_token = stripped.split()[0] + anchor_elapsed = self._elapsed_seconds(elapsed_token) or 0.0 + anchor_time = parsed_time continue - self.results.append( - { - "time_elapsed": time_elapsed, - "event": event, - "uid": uid, - "package_name": package_name, - "service": service, - } - ) + fields = stripped.split() + if not fields: + continue + if len(fields) > 1 and re.fullmatch(r"\d{2}-\d{2}", fields[0]): + time_elapsed = " ".join(fields[:2]) + line_time = self._parse_wall_time(time_elapsed) + elapsed = None + else: + time_elapsed = fields[0] + elapsed = self._elapsed_seconds(time_elapsed) + line_time = None + + timestamp = line_time + if timestamp is None and anchor_time is not None and elapsed is not None: + timestamp = anchor_time + datetime.timedelta( + seconds=elapsed - anchor_elapsed + ) + + def add( + event: str, uid: str, service: str, package_name: str | None + ) -> None: + self.results.append( + { + "time_elapsed": time_elapsed, + "timestamp": convert_datetime_to_iso(timestamp) + if timestamp + else None, + "event": event, + "uid": uid, + "package_name": package_name, + "service": service, + } + ) + + for sign, uid, raw_service in re.findall( + r"([+-])job=([^:\s]+):\"([^\"]+)\"", line + ): + service = self._normalize_service(raw_service) + add( + "start_job" if sign == "+" else "end_job", + uid, + service, + self._package_from_name(service) or service.split("/", 1)[0], + ) + + for sign, uid, package_name in re.findall( + r"([+-])top=([^:\s]+):\"([^\"]+)\"", line + ): + add( + "start_top" if sign == "+" else "end_top", + uid, + "", + package_name, + ) + + wake_match = re.search(r"\+wake_lock=([^:\s]+):\"([^\"]+)\"", line) + if wake_match: + wake_name = wake_match.group(2) + add( + "wake", + wake_match.group(1), + wake_name, + self._package_from_name(wake_name), + ) diff --git a/src/mvt/android/artifacts/dumpsys_dbinfo.py b/src/mvt/android/artifacts/dumpsys_dbinfo.py index 050582d..0972b43 100644 --- a/src/mvt/android/artifacts/dumpsys_dbinfo.py +++ b/src/mvt/android/artifacts/dumpsys_dbinfo.py @@ -29,16 +29,27 @@ class DumpsysDBInfoArtifact(AndroidArtifact): def parse(self, output: str) -> None: rxp = re.compile( - r".*\[((?:[0-9]{4}-)?[0-9]{2}-[0-9]{2} " - r"[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\]\s*" - r"(?:\[Pid:\((\d+)\)\])?([\w-]+).*?sql=\"(.+?)\"" - ) # pylint: disable=line-too-long + r"^\s*\d+:\s*\[((?:\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\]\s*" + r"(?:\[Pid:\((\d+)\)\])?\s*([\w-]+) took (\d+)ms - ([^,]+)," + r"\s*sql=\"(.*)\"(?:, path=(.*))?$" + ) - pool = None + pool: str | None = None + connection_number: int | None = None + is_primary: bool | None = None in_operations = False for line in output.splitlines(): if line.startswith("Connection pool for "): pool = line.replace("Connection pool for ", "").rstrip(":") + in_operations = False + + connection_match = re.match(r"\s+Connection #(\d+):", line) + if connection_match: + connection_number = int(connection_match.group(1)) + is_primary = None + + if line.strip().startswith("isPrimaryConnection:"): + is_primary = line.strip().split(":", 1)[1].strip() == "true" if not pool: continue @@ -52,7 +63,6 @@ class DumpsysDBInfoArtifact(AndroidArtifact): if not line.startswith(" "): in_operations = False - pool = None continue match = rxp.match(line) @@ -60,11 +70,15 @@ class DumpsysDBInfoArtifact(AndroidArtifact): continue result = { - "isodate": match.group(1), + "timestamp": match.group(1), + "pid": int(match.group(2)) if match.group(2) else None, "action": match.group(3), - "sql": match.group(4), - "path": pool, + "duration_ms": int(match.group(4)), + "status": match.group(5), + "sql": match.group(6), + "path": match.group(7) or pool, + "pool_path": pool, + "connection_number": connection_number, + "is_primary": is_primary, } - if match.group(2): - result["pid"] = match.group(2) self.results.append(result) diff --git a/src/mvt/android/artifacts/dumpsys_package_activities.py b/src/mvt/android/artifacts/dumpsys_package_activities.py index 31c6bbd..443d18f 100644 --- a/src/mvt/android/artifacts/dumpsys_package_activities.py +++ b/src/mvt/android/artifacts/dumpsys_package_activities.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ from .artifact import AndroidArtifact +from .package_resolvers import parse_resolver_table class DumpsysPackageActivitiesArtifact(AndroidArtifact): @@ -19,67 +20,11 @@ class DumpsysPackageActivitiesArtifact(AndroidArtifact): ) continue - def parse(self, content: str): + def parse(self, content: str) -> None: """ Parse the Dumpsys Package section for activities Adds results to self.results :param content: content of the package section (string) """ - self.results = [] - - in_activity_resolver_table = False - in_non_data_actions = False - intent = None - for line in content.splitlines(): - if line.startswith("Activity Resolver Table:"): - in_activity_resolver_table = True - continue - - if not in_activity_resolver_table: - continue - - if line.startswith(" Non-Data Actions:"): - in_non_data_actions = True - continue - - if not in_non_data_actions: - continue - - # If we hit an empty line, the Non-Data Actions section should be - # finished. - if line.strip() == "": - break - - # We detect the action name. - if ( - line.startswith(" " * 6) - and not line.startswith(" " * 8) - and ":" in line - ): - intent = line.strip().replace(":", "") - continue - - # If we are not in an intent block yet, skip. - if not intent: - continue - - # If we are in a block but the line does not start with 8 spaces - # it means the block ended a new one started, so we reset and - # continue. - if not line.startswith(" " * 8): - intent = None - continue - - # If we got this far, we are processing receivers for the - # activities we are interested in. - activity = line.strip().split(" ")[1] - package_name = activity.split("/")[0] - - self.results.append( - { - "intent": intent, - "package_name": package_name, - "activity": activity, - } - ) + self.results = parse_resolver_table(content, "Activity") diff --git a/src/mvt/android/artifacts/dumpsys_packages.py b/src/mvt/android/artifacts/dumpsys_packages.py index bef7744..1194a75 100644 --- a/src/mvt/android/artifacts/dumpsys_packages.py +++ b/src/mvt/android/artifacts/dumpsys_packages.py @@ -1,10 +1,10 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021-2023 The MVT Authors. +# Copyright (c) 2021-2026 The MVT Authors. # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ import re -from typing import Any, Dict, List, Optional +from typing import Any from mvt.android.utils import ROOT_PACKAGES from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult @@ -12,222 +12,244 @@ from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult from .artifact import AndroidArtifact +def _value(raw: str) -> Any: + if raw == "null": + return None + if raw in ("true", "false"): + return raw == "true" + try: + return int(raw) + except ValueError: + return raw + + class DumpsysPackagesArtifact(AndroidArtifact): def check_indicators(self) -> None: alerted_root_packages = set() for result in self.results: - if result["package_name"] in ROOT_PACKAGES: - if result["package_name"] in alerted_root_packages: - continue - alerted_root_packages.add(result["package_name"]) + package_name = result["package_name"] + if ( + package_name in ROOT_PACKAGES + and package_name not in alerted_root_packages + ): + alerted_root_packages.add(package_name) self.alertstore.medium( - f'Found an installed package related to rooting/jailbreaking: "{result["package_name"]}"', + f'Found an installed package related to rooting/jailbreaking: "{package_name}"', "", result, ) - continue - if not self.indicators: continue - - ioc_match = self.indicators.check_app_id(result.get("package_name", "")) + ioc_match = self.indicators.check_app_id(package_name) if ioc_match: self.alertstore.critical( ioc_match.message, "", result, matched_indicator=ioc_match.ioc ) def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: - records = [] timestamps = [ - {"event": "package_install", "timestamp": record["timestamp"]}, + ("package_install", record.get("timestamp")), + ("package_last_update", record.get("last_update_time")), + ] + timestamps.extend( + ("package_first_install", user.get("first_install_time")) + for user in record.get("users", []) + ) + return [ { - "event": "package_first_install", - "timestamp": record["first_install_time"], - }, - {"event": "package_last_update", "timestamp": record["last_update_time"]}, + "timestamp": timestamp, + "module": self.__class__.__name__, + "event": event, + "data": f"Install or update of package {record['package_name']}", + } + for event, timestamp in timestamps + if timestamp ] - for timestamp in timestamps: - records.append( - { - "timestamp": timestamp["timestamp"], - "module": self.__class__.__name__, - "event": timestamp["event"], - "data": f"Install or update of package {record['package_name']}", - } - ) - - return records - @staticmethod - def parse_dumpsys_package_for_details(output: str) -> Dict[str, Any]: - """ - Parse one entry of a dumpsys package information - """ - details: Dict[str, Any] = { - "uid": "", - "version_name": "", - "version_code": "", - "timestamp": "", - "first_install_time": "", - "last_update_time": "", - "installer": "", - "system": False, - "permissions": list(), - "requested_permissions": list(), + def _permission(line: str, permission_type: str) -> dict: + name, _, details = line.strip().partition(":") + granted_match = re.search(r"granted=(true|false)", details) + flags_match = re.search(r"flags=\[\s*([^]]*)\]", details) + return { + "name": name, + "type": permission_type, + "granted": granted_match.group(1) == "true" if granted_match else None, + "flags": [ + flag.strip() + for flag in (flags_match.group(1).split("|") if flags_match else []) + if flag.strip() + ], } - in_install_permissions = False - in_runtime_permissions = False - in_declared_permissions = False - in_requested_permissions = True - current_user: Optional[int] = None - first_install_times: Dict[Optional[int], str] = {} - runtime_permissions: Dict[Optional[int], List[Dict[str, Any]]] = {} + + @classmethod + def parse_dumpsys_package_for_details(cls, output: str) -> dict[str, Any]: + details: dict[str, Any] = { + "app_id": None, + "version_name": None, + "version_code": None, + "min_sdk": None, + "target_sdk": None, + "timestamp": None, + "last_update_time": None, + "installer": None, + "system": False, + "permissions": [], + "requested_permissions": [], + "users": [], + } + permission_section: str | None = None + current_user: dict[str, Any] | None = None + legacy_first_install: str | None = None + for line in output.splitlines(): - user_match = re.match(r"User (\d+):", line.strip()) + stripped = line.strip() + user_match = re.match(r"User (\d+):\s*(.*)", stripped) if user_match: - current_user = int(user_match.group(1)) + current_user = {"user_id": int(user_match.group(1)), "permissions": []} + for key, raw in re.findall(r"(\w+)=([^\s]+)", user_match.group(2)): + clean_key = { + "notLaunched": "not_launched", + "installReason": "install_reason", + "uninstallReason": "uninstall_reason", + "dataDir": "data_dir", + }.get(key, re.sub(r"(? List[Dict[str, Any]]: - """ - Parse the dumpsys package service data - """ - pkg_rxp = re.compile(r" Package \[(.+?)\].*") - - results = [] - package_name = None - package = {} - lines: list[str] = [] - for line in output.splitlines(): - if line.startswith(" Package ["): - if len(lines) > 0: - details = self.parse_dumpsys_package_for_details("\n".join(lines)) - package.update(details) - results.append(package) - lines = [] - package = {} - - matches = pkg_rxp.findall(line) - if not matches: + if current_user is not None: + user_property = re.match( + r"(installReason|uninstallReason|dataDir|firstInstallTime)=(.*)", + stripped, + ) + if user_property: + key = { + "installReason": "install_reason", + "uninstallReason": "uninstall_reason", + "dataDir": "data_dir", + "firstInstallTime": "first_install_time", + }[user_property.group(1)] + current_user[key] = _value(user_property.group(2)) continue - package_name = matches[0] - package["package_name"] = package_name + if permission_section == "requested" and line.startswith(" "): + details["requested_permissions"].append(stripped) + continue + if permission_section in ("declared", "install") and line.startswith( + " " + ): + details["permissions"].append( + cls._permission(stripped, permission_section) + ) + continue + if ( + permission_section == "runtime" + and line.startswith(" ") + and current_user is not None + ): + current_user["permissions"].append(cls._permission(stripped, "runtime")) continue - if not package_name: + simple_match = re.match( + r"(appId|userId|versionName|timeStamp|lastUpdateTime|installerPackageName)=(.*)", + stripped, + ) + if simple_match: + key = { + "appId": "app_id", + "userId": "app_id", + "versionName": "version_name", + "timeStamp": "timestamp", + "lastUpdateTime": "last_update_time", + "installerPackageName": "installer", + }[simple_match.group(1)] + raw_value = simple_match.group(2) + details[key] = ( + _value(raw_value) + if key == "app_id" or raw_value == "null" + else raw_value + ) continue + if stripped.startswith("pkgFlags="): + details["system"] = "SYSTEM" in stripped.split("=", 1)[1].split() + continue + version_match = re.match( + r"versionCode=([^\s]+)(?:\s+minSdk=([^\s]+))?(?:\s+targetSdk=([^\s]+))?", + stripped, + ) + if version_match: + details["version_code"] = _value(version_match.group(1)) + details["min_sdk"] = ( + _value(version_match.group(2)) if version_match.group(2) else None + ) + details["target_sdk"] = ( + _value(version_match.group(3)) if version_match.group(3) else None + ) + elif stripped.startswith("firstInstallTime="): + legacy_first_install = stripped.split("=", 1)[1] - lines.append(line) + if legacy_first_install: + user_zero = next( + (user for user in details["users"] if user["user_id"] == 0), None + ) + if user_zero is None: + user_zero = {"user_id": 0, "permissions": []} + details["users"].append(user_zero) + user_zero.setdefault("first_install_time", legacy_first_install) + return details - if len(lines) > 0: - details = self.parse_dumpsys_package_for_details("\n".join(lines)) - package.update(details) - results.append(package) + def parse(self, content: str) -> None: + self.results: list[dict[str, Any]] = [] + category: str | None = None + package: dict[str, Any] | None = None + block: list[str] = [] - return results + def finish() -> None: + nonlocal package, block + if package is not None: + package.update(self.parse_dumpsys_package_for_details("\n".join(block))) + self.results.append(package) + package = None + block = [] - def parse(self, content: str): - """ - Parse the Dumpsys Package section for activities - Adds results to self.results - - :param content: content of the package section (string) - """ - self.results = [] - package = [] - - in_package_list = False for line in content.splitlines(): - if line.startswith("Packages:"): - in_package_list = True + if line == "Packages:": + finish() + category = "active" continue - - if not in_package_list: + if line == "Hidden system packages:": + finish() + category = "hidden_system" continue - - if line.strip() == "": - break - - package.append(line) - - self.results = self.parse_dumpsys_packages("\n".join(package)) + package_match = re.match(r"^ Package \[([^]]+)\]", line) + if package_match and category: + finish() + package = { + "package_name": package_match.group(1), + "package_type": category, + } + continue + if ( + category + and line + and not line.startswith(" ") + and not line.endswith(" overlay paths:") + ): + finish() + category = None + continue + if package is not None: + block.append(line) + finish() diff --git a/src/mvt/android/artifacts/dumpsys_platform_compat.py b/src/mvt/android/artifacts/dumpsys_platform_compat.py index c01fbfd..08cb46e 100644 --- a/src/mvt/android/artifacts/dumpsys_platform_compat.py +++ b/src/mvt/android/artifacts/dumpsys_platform_compat.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import re + from .artifact import AndroidArtifact @@ -25,19 +27,37 @@ class DumpsysPlatformCompatArtifact(AndroidArtifact): def parse(self, data: str) -> None: for line in data.splitlines(): - if not line.startswith("ChangeId(168419799; name=DOWNSCALED;"): + match = re.match(r"ChangeId\((\d+);\s*(.*)\)$", line.strip()) + if not match or "rawOverrides={" not in line: continue - - if line.strip() == "": - break - - # Look for rawOverrides field - if "rawOverrides={" in line: - # Extract the content inside the braces for rawOverrides - overrides_field = line.split("rawOverrides={", 1)[1].split("};", 1)[0] - - for entry in overrides_field.split(", "): - # Extract app name - uninstall_app = entry.split("=")[0].strip() - - self.results.append({"package_name": uninstall_app}) + body = match.group(2) + name_match = re.search(r"(?:^|;\s*)name=([^;]+)", body) + state = ( + "enabled" + if re.search(r"(?:^|;\s*)enabled(?:;|$)", body) + else "disabled" + ) + overridable = bool(re.search(r"(?:^|;\s*)overridable(?:;|$)", body)) + overrides_field = body.split("rawOverrides={", 1)[1].split("}", 1)[0] + for entry in overrides_field.split(","): + package_name, separator, raw_value = entry.strip().partition("=") + if not separator: + continue + value: bool | int | str + if raw_value in ("true", "false"): + value = raw_value == "true" + else: + try: + value = int(raw_value) + except ValueError: + value = raw_value + self.results.append( + { + "change_id": int(match.group(1)), + "change_name": name_match.group(1) if name_match else None, + "change_state": state, + "overridable": overridable, + "package_name": package_name, + "override_value": value, + } + ) diff --git a/src/mvt/android/artifacts/dumpsys_receivers.py b/src/mvt/android/artifacts/dumpsys_receivers.py index b437930..7594335 100644 --- a/src/mvt/android/artifacts/dumpsys_receivers.py +++ b/src/mvt/android/artifacts/dumpsys_receivers.py @@ -4,6 +4,7 @@ # https://license.mvt.re/1.1/ from .artifact import AndroidArtifact +from .package_resolvers import parse_resolver_table INTENT_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS" INTENT_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED" @@ -18,115 +19,45 @@ class DumpsysReceiversArtifact(AndroidArtifact): """ def check_indicators(self) -> None: - for intent, receivers in self.results.items(): - for receiver in receivers: - if intent == INTENT_NEW_OUTGOING_SMS: - self.log.info( - 'Found a receiver to intercept outgoing SMS messages: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_SMS_RECEIVED: - self.log.info( - 'Found a receiver to intercept incoming SMS messages: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_DATA_SMS_RECEIVED: - self.log.info( - 'Found a receiver to intercept incoming data SMS message: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_PHONE_STATE: - self.log.info( - "Found a receiver monitoring " - 'telephony state/incoming calls: "%s"', - receiver["receiver"], - ) - elif intent == INTENT_NEW_OUTGOING_CALL: - self.log.info( - 'Found a receiver monitoring outgoing calls: "%s"', - receiver["receiver"], - ) + for receiver in self.results: + intent = receiver["key"] + if intent == INTENT_NEW_OUTGOING_SMS: + self.log.info( + 'Found a receiver to intercept outgoing SMS messages: "%s"', + receiver["component"], + ) + elif intent == INTENT_SMS_RECEIVED: + self.log.info( + 'Found a receiver to intercept incoming SMS messages: "%s"', + receiver["component"], + ) + elif intent == INTENT_DATA_SMS_RECEIVED: + self.log.info( + 'Found a receiver to intercept incoming data SMS message: "%s"', + receiver["component"], + ) + elif intent == INTENT_PHONE_STATE: + self.log.info( + 'Found a receiver monitoring telephony state/incoming calls: "%s"', + receiver["component"], + ) + elif intent == INTENT_NEW_OUTGOING_CALL: + self.log.info( + 'Found a receiver monitoring outgoing calls: "%s"', + receiver["component"], + ) - if not self.indicators: - continue + if not self.indicators: + continue - ioc_match = self.indicators.check_app_id(receiver["package_name"]) - if ioc_match: - self.alertstore.critical( - ioc_match.message, - "", - {intent: receiver}, - matched_indicator=ioc_match.ioc, - ) - continue + ioc_match = self.indicators.check_app_id(receiver["package_name"]) + if ioc_match: + self.alertstore.critical( + ioc_match.message, + "", + receiver, + matched_indicator=ioc_match.ioc, + ) def parse(self, output: str) -> None: - self.results: dict[str, list[dict[str, str]]] = {} - - in_receiver_resolver_table = False - in_non_data_actions = False - intent = None - for line in output.splitlines(): - if line.startswith("Receiver Resolver Table:"): - in_receiver_resolver_table = True - continue - - if not in_receiver_resolver_table: - continue - - if line.startswith(" Non-Data Actions:"): - in_non_data_actions = True - continue - - if not in_non_data_actions: - continue - - # If we hit an empty line, the Non-Data Actions section should be - # finished. - if line.strip() == "": - break - - # We detect the action name. - if ( - line.startswith(" " * 6) - and not line.startswith(" " * 8) - and ":" in line - ): - intent = line.strip().replace(":", "") - self.results[intent] = [] - continue - - parts = line.strip().split(" ") - if len(parts) < 2: - # A single-token line here is not a receiver. Real dumpstate - # output can print an action header mis-indented (observed with - # 15 leading spaces instead of 6), which used to raise - # IndexError and abort the whole module. Treat a trailing-colon - # token as the next action, skip anything else. - if parts[0].endswith(":"): - intent = parts[0][:-1] - self.results.setdefault(intent, []) - continue - - # If we are not in an intent block yet, skip. - if not intent: - continue - - # If we are in a block but the line does not start with 8 spaces - # it means the block ended a new one started, so we reset and - # continue. - if not line.startswith(" " * 8): - intent = None - continue - - # If we got this far, we are processing receivers for the - # activities we are interested in. - receiver = parts[1] - package_name = receiver.split("/")[0] - - self.results[intent].append( - { - "package_name": package_name, - "receiver": receiver, - } - ) + self.results = parse_resolver_table(output, "Receiver") diff --git a/src/mvt/android/artifacts/getprop.py b/src/mvt/android/artifacts/getprop.py index debcfc4..094eb5d 100644 --- a/src/mvt/android/artifacts/getprop.py +++ b/src/mvt/android/artifacts/getprop.py @@ -28,19 +28,16 @@ INTERESTING_PROPERTIES = [ class GetProp(AndroidArtifact): def parse(self, entry: str) -> None: self.results: List[Dict[str, str]] = [] - rxp = re.compile(r"\[(.+?)\]: \[(.+?)\]") + # A property value may span several lines: persist.sys.boot.reason.history + # prints one boot per line. Matching the whole section instead of line by + # line lets a value run to the first closing bracket that ends a line. + rxp = re.compile( + r"^[ \t]*\[([^]]+)\]: \[(.*?)\][ \t\r]*$", + re.MULTILINE | re.DOTALL, + ) - for line in entry.splitlines(): - line = line.strip() - if line == "": - continue - - matches = re.findall(rxp, line) - if not matches or len(matches[0]) != 2: - continue - - prop_entry = {"name": matches[0][0], "value": matches[0][1]} - self.results.append(prop_entry) + for name, value in rxp.findall(entry): + self.results.append({"name": name, "value": value}) def get_device_timezone(self) -> str | None: """ diff --git a/src/mvt/android/artifacts/mounts.py b/src/mvt/android/artifacts/mounts.py index a35d889..5a5537b 100644 --- a/src/mvt/android/artifacts/mounts.py +++ b/src/mvt/android/artifacts/mounts.py @@ -117,6 +117,51 @@ class Mounts(AndroidArtifact): # Skip lines that don't match expected format continue + @staticmethod + def parse_mountinfo(entry: str, process_id: int) -> list[dict[str, Any]]: + """Parse Linux /proc/PID/mountinfo records.""" + results = [] + for line in entry.splitlines(): + fields = line.split() + if "-" not in fields: + continue + separator = fields.index("-") + if separator < 6 or len(fields) < separator + 4: + continue + try: + mount_id = int(fields[0]) + parent_id = int(fields[1]) + except ValueError: + continue + mount_options = fields[5].split(",") + super_options = fields[separator + 3].split(",") + options = list(dict.fromkeys(mount_options + super_options)) + mount_point = fields[4].replace("\\040", " ") + device = fields[separator + 2].replace("\\040", " ") + filesystem_type = fields[separator + 1] + is_system = mount_point in SUSPICIOUS_MOUNT_POINTS or any( + mount_point.startswith(f"{prefix}/") + for prefix in SUSPICIOUS_MOUNT_POINTS + ) + results.append( + { + "mount_id": mount_id, + "parent_id": parent_id, + "major_minor": fields[2], + "root": fields[3].replace("\\040", " "), + "mount_point": mount_point, + "device": device, + "filesystem_type": filesystem_type, + "mount_options": ",".join(options), + "options_list": options, + "optional_fields": fields[6:separator], + "is_system_partition": is_system, + "is_read_write": "rw" in options, + "process_ids": [process_id], + } + ) + return results + def check_indicators(self) -> None: """ Check for suspicious mount configurations that may indicate root access diff --git a/src/mvt/android/artifacts/package_resolvers.py b/src/mvt/android/artifacts/package_resolvers.py new file mode 100644 index 0000000..60c2212 --- /dev/null +++ b/src/mvt/android/artifacts/package_resolvers.py @@ -0,0 +1,70 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import re + + +RESOLVER_TYPES = { + "Full MIME Types": "full_mime_type", + "Base MIME Types": "base_mime_type", + "Wild MIME Types": "wild_mime_type", + "Schemes": "scheme", + "Non-Data Actions": "non_data_action", + "MIME Typed Actions": "mime_typed_action", +} + +ENTRY_RE = re.compile( + r"^\s{8,}[0-9a-fA-F]+\s+(?P\S+)" + r"(?:\s+\((?P\d+)\s+filters?\))?\s*$" +) + + +def parse_resolver_table(content: str, table_name: str) -> list[dict]: + """Parse every resolver category from a dumpsys package table.""" + results: list[dict] = [] + in_table = False + resolver_type: str | None = None + key: str | None = None + + for line in content.splitlines(): + if line.startswith(f"{table_name} Resolver Table:"): + in_table = True + continue + if not in_table: + continue + + if line and not line.startswith(" "): + break + + section_match = re.match(r"^ {2}([^ ].*):\s*$", line) + if section_match: + resolver_type = RESOLVER_TYPES.get(section_match.group(1)) + key = None + continue + if resolver_type is None: + continue + + key_match = re.match(r"^ {6,}([^ ].*):\s*$", line) + if key_match: + key = key_match.group(1) + continue + if key is None: + continue + + entry_match = ENTRY_RE.match(line) + if not entry_match: + continue + component = entry_match.group("component") + results.append( + { + "resolver_type": resolver_type, + "key": key, + "package_name": component.split("/", 1)[0], + "component": component, + "filter_count": int(entry_match.group("filters") or 1), + } + ) + + return results diff --git a/src/mvt/android/artifacts/processes.py b/src/mvt/android/artifacts/processes.py index 5471eb2..97e27a9 100644 --- a/src/mvt/android/artifacts/processes.py +++ b/src/mvt/android/artifacts/processes.py @@ -1,72 +1,91 @@ # Mobile Verification Toolkit (MVT) -# Copyright (c) 2021-2023 The MVT Authors. +# Copyright (c) 2021-2026 The MVT Authors. # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ from .artifact import AndroidArtifact +FIELD_NAMES = { + "LABEL": "label", + "USER": "user", + "PID": "pid", + "TID": "tid", + "PPID": "ppid", + "VSZ": "virtual_memory_size", + "RSS": "resident_set_size", + "WCHAN": "wchan", + "ADDR": "address", + "S": "state", + "PRI": "priority", + "NI": "nice", + "RTPRIO": "realtime_priority", + "SCH": "scheduler", + "PCY": "policy", + "TIME": "cpu_time", + "CMD": "command", + "NAME": "command", +} +INTEGER_FIELDS = { + "pid", + "tid", + "ppid", + "virtual_memory_size", + "resident_set_size", + "priority", + "nice", +} + + class Processes(AndroidArtifact): def parse(self, entry: str) -> None: - for line in entry.splitlines()[1:]: - proc = line.split() + self.results = [] + lines = [line for line in entry.splitlines() if line.strip()] + if not lines: + return + headers = lines[0].split() + if not all(header in FIELD_NAMES for header in headers): + return - # Skip empty lines - if len(proc) == 0: + for line in lines[1:]: + values = line.split(None, len(headers) - 1) + if len(values) != len(headers): continue - - # Sometimes WCHAN is empty. - if len(proc) == 8: - proc = proc[:5] + [""] + proc[5:] - - # Sometimes there is the security label. - if proc[0].startswith("u:r"): - label = proc[0] - proc = proc[1:] - else: - label = "" - - # Sometimes there is no WCHAN. - if len(proc) < 9: - proc = proc[:5] + [""] + proc[5:] - - self.results.append( - { - "user": proc[0], - "pid": int(proc[1]), - "ppid": int(proc[2]), - "virtual_memory_size": int(proc[3]), - "resident_set_size": int(proc[4]), - "wchan": proc[5], - "aprocress": proc[6], - "stat": proc[7], - "proc_name": proc[8].strip("[]"), - "label": label, - } - ) + result = {} + valid = True + for header, raw in zip(headers, values): + key = FIELD_NAMES[header] + value: str | int = raw.strip("[]") if key == "command" else raw + if key in INTEGER_FIELDS: + try: + value = int(value) + except ValueError: + valid = False + break + result[key] = value + if valid: + self.results.append(result) def check_indicators(self) -> None: if not self.indicators: return - for result in self.results: - proc_name = result.get("proc_name", "") - if not proc_name: + command = result.get("command", "") + if not isinstance(command, str): continue - - # Skipping this process because of false positives. - if result["proc_name"] == "gatekeeperd": + process_name = command.rsplit("/", 1)[-1] + if not process_name or process_name == "gatekeeperd": continue - - ioc_match = self.indicators.check_app_id(proc_name) - if ioc_match: - self.alertstore.critical( - ioc_match.message, "", result, matched_indicator=ioc_match.ioc - ) - continue - - ioc_match = self.indicators.check_process(proc_name) - if ioc_match: - self.alertstore.critical( - ioc_match.message, "", result, matched_indicator=ioc_match.ioc - ) + for checker in ( + self.indicators.check_app_id, + self.indicators.check_process, + ): + ioc_match = checker(process_name) + if ioc_match: + self.alertstore.critical( + ioc_match.message, + "", + result, + matched_indicator=ioc_match.ioc, + ) + break diff --git a/src/mvt/android/artifacts/settings.py b/src/mvt/android/artifacts/settings.py index 17c3bf2..e0cd5f0 100644 --- a/src/mvt/android/artifacts/settings.py +++ b/src/mvt/android/artifacts/settings.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import re + from .artifact import AndroidArtifact ANDROID_DANGEROUS_SETTINGS = [ @@ -60,6 +62,28 @@ ANDROID_DANGEROUS_SETTINGS = [ class Settings(AndroidArtifact): + def parse(self, content: str) -> None: + self.results: dict[str, dict[str, str]] = {} + namespace: str | None = None + for line in content.splitlines(): + heading = re.match( + r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$", + line.strip(), + ) + if heading: + namespace = f"{heading.group(1).lower()}:user_{heading.group(2)}" + self.results[namespace] = {} + continue + if namespace is None or not line.startswith("_id:"): + continue + setting = re.match( + r"^_id:\S+\s+name:(.*?)\s+pkg:.*?\s+value:(.*?)" + r"(?:\s+default:.*\s+defaultSystemSet:(?:true|false))?$", + line, + ) + if setting: + self.results[namespace][setting.group(1)] = setting.group(2) + def check_indicators(self) -> None: for namespace, settings in self.results.items(): for key, value in settings.items(): diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index bcfb389..c749d1a 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -131,6 +131,13 @@ class TombstoneCrashArtifact(AndroidArtifact): self, file_name: str, file_timestamp: datetime.datetime, data: bytes ) -> None: """Parse Android tombstone crash files from a protobuf object.""" + self.results.append(self.parse_protobuf_record(file_name, file_timestamp, data)) + + def parse_protobuf_record( + self, file_name: str, file_timestamp: datetime.datetime, data: bytes + ) -> dict: + if not data: + raise ValueError("empty protobuf tombstone") tombstone_pb = Tombstone().parse(data) tombstone_dict = tombstone_pb.to_dict( casing=betterproto2.Casing.SNAKE, include_default_values=True @@ -143,20 +150,31 @@ class TombstoneCrashArtifact(AndroidArtifact): tombstone_dict["file_name"] = file_name tombstone_dict["file_timestamp"] = convert_datetime_to_iso(file_timestamp) tombstone_dict["process_name"] = self._proccess_name_from_thread(tombstone_dict) + if isinstance(tombstone_dict.get("selinux_label"), str): + tombstone_dict["selinux_label"] = tombstone_dict["selinux_label"].rstrip( + "\x00" + ) # Confirm the tombstone is valid, and matches the output model tombstone = TombstoneCrashResult.model_validate(tombstone_dict) - self.results.append(tombstone.model_dump()) + return tombstone.model_dump() def parse( self, file_name: str, file_timestamp: datetime.datetime, content: bytes ) -> None: """Parse text Android tombstone crash files.""" + self.results.append(self.parse_text_record(file_name, file_timestamp, content)) + + def parse_text_record( + self, file_name: str, file_timestamp: datetime.datetime, content: bytes + ) -> dict: + if not content: + raise ValueError("empty plaintext tombstone") tombstone_dict = { "file_name": file_name, "file_timestamp": convert_datetime_to_iso(file_timestamp), } - lines = content.decode("utf-8").splitlines() + lines = content.decode("utf-8", errors="replace").splitlines() for line_num, line in enumerate(lines, 1): if not line.strip() or TOMBSTONE_DELIMITER in line: continue @@ -171,7 +189,7 @@ class TombstoneCrashArtifact(AndroidArtifact): # Validate the tombstone and add it to the results tombstone = TombstoneCrashResult.model_validate(tombstone_dict) - self.results.append(tombstone.model_dump()) + return tombstone.model_dump() def _parse_tombstone_line( self, line: str, key: str, destination_key: str, tombstone: dict @@ -191,11 +209,18 @@ class TombstoneCrashArtifact(AndroidArtifact): def _load_key_value_line( self, line: str, key: str, destination_key: str, tombstone: dict ) -> bool: - line_key, value = line.split(":", 1) - if line_key != key: - raise ValueError(f"Expected key {key}, got {line_key}") + # The caller matched the key as a bare prefix, so a longer word starting + # with it arrives here: `Caused by: …` inside an abort message reaches + # the `Cause` key. That is a different line, not a broken file — say so + # by declining it, and let the remaining keys have their turn. Raising + # here discarded the whole tombstone, crash and stack trace included. + line_key, separator, value = line.partition(":") + if not separator or line_key != key: + return False - value_clean = value.strip().strip("'") + value_clean = value.strip() + if len(value_clean) >= 2 and value_clean[0] == value_clean[-1] == "'": + value_clean = value_clean[1:-1] if destination_key == "uid": tombstone[destination_key] = int(value_clean) elif destination_key == "process_uptime": @@ -269,9 +294,7 @@ class TombstoneCrashArtifact(AndroidArtifact): @staticmethod def _parse_timestamp_string(timestamp: str) -> str: timestamp_parsed = parser.parse(timestamp) - # Preserve the source wall-clock time while returning the project-wide ISO format. - local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc) - return convert_datetime_to_iso(local_timestamp) + return convert_datetime_to_iso(timestamp_parsed) @staticmethod def _proccess_name_from_thread(tombstone_dict: dict) -> str: diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index ca7e3dc..7b78a4e 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -14,13 +14,6 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.completion import ( - SUPPORTED_SHELLS, - completion_instructions, - generate_completion_script, - install_completion_script, -) from mvt.common.help import ( HELP_MSG_ANDROID_BACKUP_PASSWORD, HELP_MSG_CHECK_ADB_REMOVED, @@ -31,7 +24,6 @@ from mvt.common.help import ( HELP_MSG_CHECK_IOCS, HELP_MSG_CHECK_INTRUSION_LOGS, HELP_MSG_DELAY_CHECKS, - HELP_MSG_COMPLETION, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_HASHES, @@ -43,23 +35,16 @@ from mvt.common.help import ( HELP_MSG_OUTPUT, HELP_MSG_STIX2, HELP_MSG_VERBOSE, + HELP_MSG_VERBOSE_COMMAND, HELP_MSG_VERSION, HELP_MSG_VIRUS_TOTAL, ) -from mvt.common.logo import logo -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import init_logging, set_verbose_logging -from .cmd_check_androidqf import CmdAndroidCheckAndroidQF -from .cmd_check_backup import CmdAndroidCheckBackup -from .cmd_check_bugreport import CmdAndroidCheckBugreport -from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs -from .modules.intrusion_logs import INTRUSION_LOGS_MODULES -from .modules.androidqf import ANDROIDQF_MODULES -from .modules.backup import BACKUP_MODULES -from .modules.backup.helpers import cli_load_android_backup_password -from .modules.bugreport import BUGREPORT_MODULES +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-android, including by shell completion on +# every keystroke, so importing it must do no more than build the command tree: +# the forensic modules, the backup parsers and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -77,7 +62,14 @@ def _get_disable_flags(ctx): ) +def _get_verbose(ctx): + """Return whether --verbose was passed to the CLI itself.""" + return bool(ctx.obj and ctx.obj.get("verbose", False)) + + def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -97,60 +89,31 @@ def _load_custom_modules(load_module): is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - if ctx.invoked_subcommand != "completion": - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) + + from mvt.common.logo import logo + + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== # Command: version # ============================================================================== -@cli.command("version", help=HELP_MSG_VERSION) +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) def version(): return -# ============================================================================== -# Command: completion -# ============================================================================== -@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) -@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) -@click.option( - "--install", - is_flag=True, - help="Write completion files and update shell configuration.", -) -@click.pass_context -def completion(ctx, shell, install): - program_name = "mvt-android" - - if shell is None: - if install: - raise click.UsageError("A shell is required when using --install.") - click.echo(completion_instructions(program_name)) - return - - root_cli = ctx.find_root().command - - if install: - script_path = install_completion_script(root_cli, program_name, shell) - click.echo(f"Installed {shell} completion to {script_path}") - if shell in ("bash", "zsh"): - click.echo(f"Updated ~/.{shell}rc") - else: - click.echo("Fish loads completion files automatically.") - return - - click.echo(generate_completion_script(root_cli, program_name, shell)) - - # ============================================================================== # Command: check-adb (removed) # ============================================================================== @@ -187,7 +150,7 @@ def check_adb(ctx): default=[], help=HELP_MSG_LOAD_MODULE, ) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BUGREPORT_PATH", type=click.Path(exists=True)) @click.pass_context def check_bugreport( @@ -200,7 +163,9 @@ def check_bugreport( verbose, bugreport_path, ): - set_verbose_logging(verbose) + from .cmd_check_bugreport import CmdAndroidCheckBugreport + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as bug reports are small. cmd = CmdAndroidCheckBugreport( @@ -255,7 +220,7 @@ def check_bugreport( ) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( @@ -269,7 +234,10 @@ def check_backup( verbose, backup_path, ): - set_verbose_logging(verbose) + from .cmd_check_backup import CmdAndroidCheckBackup + from .modules.backup.helpers import cli_load_android_backup_password + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) # Always generate hashes as backups are generally small. @@ -329,7 +297,7 @@ def check_backup( ) @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE) @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("ANDROIDQF_PATH", type=click.Path(exists=True)) @click.pass_context def check_androidqf( @@ -347,7 +315,10 @@ def check_androidqf( verbose, androidqf_path, ): - set_verbose_logging(verbose) + from .cmd_check_androidqf import CmdAndroidCheckAndroidQF + from .modules.backup.helpers import cli_load_android_backup_password + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdAndroidCheckAndroidQF( @@ -415,7 +386,7 @@ def check_androidqf( "time instead of UTC." ), ) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("LOGS_PATH", type=click.Path(exists=True)) @click.pass_context def check_intrusion_logs( @@ -429,7 +400,9 @@ def check_intrusion_logs( verbose, logs_path, ): - set_verbose_logging(verbose) + from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) module_options = {} @@ -482,6 +455,10 @@ def check_intrusion_logs( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import ANDROID_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -492,9 +469,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): custom_modules=custom_modules, platform="android", ) - cmd.modules = ( - BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES - ) + cmd.modules = ANDROID_CHECK_IOCS_MODULES if list_modules: cmd.list_modules() @@ -510,12 +485,25 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_indicators(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() -register_cli_plugins( - cli, - entry_point_group=ANDROID_CLI_PLUGIN_GROUP, - environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV, -) +# ============================================================================== +# Entry point of the mvt-android console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt-android CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=ANDROID_CLI_PLUGIN_GROUP, + environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/android/cmd_check_androidqf.py b/src/mvt/android/cmd_check_androidqf.py index ffa3e13..99ac5a3 100644 --- a/src/mvt/android/cmd_check_androidqf.py +++ b/src/mvt/android/cmd_check_androidqf.py @@ -292,9 +292,7 @@ class CmdAndroidCheckAndroidQF(Command): try: cmd.from_ab(backup) except InvalidAndroidBackup as exc: - self.log.warning( - "Skipping backup modules as backup.ab is malformed: %s", exc - ) + self.log.warning("Skipping backup modules: %s", exc) return False cmd.run() diff --git a/src/mvt/android/cmd_check_backup.py b/src/mvt/android/cmd_check_backup.py index b75bb34..94be4ee 100644 --- a/src/mvt/android/cmd_check_backup.py +++ b/src/mvt/android/cmd_check_backup.py @@ -87,11 +87,15 @@ class CmdAndroidCheckBackup(Command): if header["encryption"] != "none": password = prompt_or_load_android_backup_password(log, self.module_options) if not password: + if self.sub_command: + raise InvalidAndroidBackup("No backup password provided") log.critical("No backup password provided.") sys.exit(1) try: tardata = parse_backup_file(ab_file_bytes, password=password) except InvalidBackupPassword: + if self.sub_command: + raise InvalidAndroidBackup("Invalid backup password") log.critical("Invalid backup password") sys.exit(1) except AndroidBackupParsingError as exc: diff --git a/src/mvt/android/command_modules.py b/src/mvt/android/command_modules.py new file mode 100644 index 0000000..9b6b56e --- /dev/null +++ b/src/mvt/android/command_modules.py @@ -0,0 +1,23 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Module lists an mvt-android command composes from more than one family. + +Commands whose modules are one family read that family directly. check-iocs +re-checks stored results, so it has to know every module that could have +written one, and both the CLI and any other code needing that answer share +the list from here rather than each concatenating their own. +""" + +from mvt.common.module import MVTModule + +from .modules.androidqf import ANDROIDQF_MODULES +from .modules.backup import BACKUP_MODULES +from .modules.bugreport import BUGREPORT_MODULES +from .modules.intrusion_logs import INTRUSION_LOGS_MODULES + +ANDROID_CHECK_IOCS_MODULES: list[type[MVTModule]] = ( + BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES +) diff --git a/src/mvt/android/modules/androidqf/__init__.py b/src/mvt/android/modules/androidqf/__init__.py index 1d8d619..05e7cde 100644 --- a/src/mvt/android/modules/androidqf/__init__.py +++ b/src/mvt/android/modules/androidqf/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .aqf_files import AQFFiles from .aqf_getprop import AQFGetProp from .aqf_log_timestamps import AQFLogTimestamps @@ -12,7 +14,7 @@ from .aqf_settings import AQFSettings from .mounts import Mounts from .root_binaries import RootBinaries -ANDROIDQF_MODULES = [ +ANDROIDQF_MODULES: list[type[MVTModule]] = [ AQFPackages, AQFProcesses, AQFGetProp, diff --git a/src/mvt/android/modules/backup/__init__.py b/src/mvt/android/modules/backup/__init__.py index 47a5e1c..683a383 100644 --- a/src/mvt/android/modules/backup/__init__.py +++ b/src/mvt/android/modules/backup/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .sms import SMS -BACKUP_MODULES = [SMS] +BACKUP_MODULES: list[type[MVTModule]] = [SMS] diff --git a/src/mvt/android/modules/bugreport/__init__.py b/src/mvt/android/modules/bugreport/__init__.py index 1594af9..e23bf5a 100644 --- a/src/mvt/android/modules/bugreport/__init__.py +++ b/src/mvt/android/modules/bugreport/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .dumpsys_accessibility import DumpsysAccessibility from .dumpsys_activities import DumpsysActivities from .dumpsys_appops import DumpsysAppops @@ -16,8 +18,11 @@ from .dumpsys_receivers import DumpsysReceivers from .dumpsys_adb_state import DumpsysADBState from .fs_timestamps import BugReportTimestamps from .tombstones import Tombstones +from .mounts import Mounts +from .processes import Processes +from .settings import Settings -BUGREPORT_MODULES = [ +BUGREPORT_MODULES: list[type[MVTModule]] = [ DumpsysAccessibility, DumpsysActivities, DumpsysAppops, @@ -31,4 +36,7 @@ BUGREPORT_MODULES = [ DumpsysADBState, BugReportTimestamps, Tombstones, + Processes, + Settings, + Mounts, ] diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index 156e01c..01af8c9 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -72,7 +72,11 @@ class BugReportModule(MVTModule): if not self.extract_path: raise ValueError("extract_path is not set") joined = os.path.join(self.extract_path, file_path) - if not Path(joined).resolve().is_relative_to(Path(self.extract_path).resolve()): + if ( + not Path(joined) + .resolve() + .is_relative_to(Path(self.extract_path).resolve()) + ): raise ValueError("unsafe file_path") handle = open(joined, "rb") @@ -100,6 +104,27 @@ class BugReportModule(MVTModule): return None + @staticmethod + def extract_command_section(content: str, heading: str) -> str: + """Return a bugreport command section without consuming the next one. + + Bugreport separators include timing text, so looking for a line equal to + ``------`` is not sufficient and can accidentally feed the remainder of + dumpstate to a parser. + """ + lines: list[str] = [] + in_section = False + for line in content.splitlines(): + stripped = line.strip() + if not in_section: + if stripped.startswith(heading): + in_section = True + continue + if stripped.startswith("------"): + break + lines.append(line) + return "\n".join(lines) + def _get_file_modification_time(self, file_path: str) -> datetime.datetime: if self.zip_archive: file_timetuple = self.zip_archive.getinfo(file_path).date_time diff --git a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py index 72208c0..02d85a3 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py +++ b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py @@ -49,9 +49,7 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule): self.parse(content) for result in self.results: - self.log.info( - 'Found installed accessibility service "%s"', result.get("service") - ) + self.log.info('Found accessibility service "%s"', result.get("component")) self.log.info( "Identified a total of %d accessibility services", len(self.results) diff --git a/src/mvt/android/modules/bugreport/dumpsys_getprop.py b/src/mvt/android/modules/bugreport/dumpsys_getprop.py index 198a1d2..b9c20ea 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_getprop.py +++ b/src/mvt/android/modules/bugreport/dumpsys_getprop.py @@ -44,21 +44,8 @@ class DumpsysGetProp(GetPropArtifact, BugReportModule): ) return - lines = [] - in_getprop = False - - for line in content.decode(errors="ignore").splitlines(): - if line.strip().startswith("------ SYSTEM PROPERTIES"): - in_getprop = True - continue - - if not in_getprop: - continue - - if line.strip() == "------": - break - - lines.append(line) - - self.parse("\n".join(lines)) + section = self.extract_command_section( + content.decode(errors="ignore"), "------ SYSTEM PROPERTIES" + ) + self.parse(section) self.log.info("Extracted %d Android system properties", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/dumpsys_packages.py b/src/mvt/android/modules/bugreport/dumpsys_packages.py index aebec4c..ff944e3 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_packages.py +++ b/src/mvt/android/modules/bugreport/dumpsys_packages.py @@ -50,7 +50,10 @@ class DumpsysPackages(DumpsysPackagesArtifact, BugReportModule): for result in self.results: dangerous_permissions_count = 0 - for perm in result["permissions"]: + permissions = list(result["permissions"]) + for user in result.get("users", []): + permissions.extend(user.get("permissions", [])) + for perm in permissions: if perm["name"] in DANGEROUS_PERMISSIONS: dangerous_permissions_count += 1 diff --git a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py index 968bc25..dd06b40 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py +++ b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py @@ -13,7 +13,7 @@ from mvt.common.module_types import ModuleResults class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): - """This module extracts details on uninstalled apps.""" + """This module extracts raw per-package compatibility overrides.""" def __init__( self, @@ -48,4 +48,4 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): ) self.parse(content) - self.log.info("Found %d uninstalled apps", len(self.results)) + self.log.info("Found %d package compatibility overrides", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/dumpsys_receivers.py b/src/mvt/android/modules/bugreport/dumpsys_receivers.py index 8907d5e..a82f021 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_receivers.py +++ b/src/mvt/android/modules/bugreport/dumpsys_receivers.py @@ -33,7 +33,7 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule): results=results, ) - self.results = results if results else {} + self.results = results if results else [] def run(self) -> None: content = self._get_dumpstate_file() @@ -49,4 +49,4 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule): ) self.parse(dumpsys_section) - self.log.info("Extracted receivers for %d intents", len(self.results)) + self.log.info("Extracted %d package receivers", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/fs_timestamps.py b/src/mvt/android/modules/bugreport/fs_timestamps.py index 5a2ca48..3c7e324 100644 --- a/src/mvt/android/modules/bugreport/fs_timestamps.py +++ b/src/mvt/android/modules/bugreport/fs_timestamps.py @@ -4,12 +4,15 @@ # https://license.mvt.re/1.1/ import logging +import datetime from typing import Optional +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from mvt.common.utils import convert_datetime_to_iso from .base import BugReportModule from mvt.common.module_types import ModuleResults from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact +from mvt.android.artifacts.getprop import GetProp class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): @@ -38,15 +41,44 @@ class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): def run(self) -> None: filesystem_files = self._get_files_by_pattern("FS/*") + timezone_name = None + dumpstate = self._get_dumpstate_file() + if dumpstate: + section = self.extract_command_section( + dumpstate.decode("utf-8", errors="replace"), + "------ SYSTEM PROPERTIES", + ) + properties = GetProp() + properties.parse(section) + timezone_name = properties.get_device_timezone() + timezone = None + if timezone_name: + try: + timezone = ZoneInfo(timezone_name) + except ZoneInfoNotFoundError: + self.log.warning("Unknown device timezone %s", timezone_name) + self.results = [] for file in filesystem_files: # Only the modification time is available in the zip file metadata. # The timezone is the local timezone of the machine the phone. modification_time = self._get_file_modification_time(file) + utc_time = None + if timezone is not None: + utc_time = convert_datetime_to_iso( + modification_time.replace(tzinfo=timezone).astimezone( + datetime.timezone.utc + ) + ) self.results.append( { "path": file, "modified_time": convert_datetime_to_iso(modification_time), + "modified_time_utc": utc_time, + "timezone": timezone_name, + "timestamp_source": ( + "zip_metadata" if self.zip_archive else "filesystem_metadata" + ), } ) diff --git a/src/mvt/android/modules/bugreport/mounts.py b/src/mvt/android/modules/bugreport/mounts.py new file mode 100644 index 0000000..ecef40c --- /dev/null +++ b/src/mvt/android/modules/bugreport/mounts.py @@ -0,0 +1,41 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. + +import re + +from mvt.android.artifacts.mounts import Mounts as MountsArtifact + +from .base import BugReportModule + + +class Mounts(MountsArtifact, BugReportModule): + """Extract and deduplicate process mount namespaces from mountinfo.""" + + def run(self) -> None: + mount_files = self._get_files_by_pattern("FS/proc/*/mountinfo") + if not mount_files: + mount_files = self._get_files_by_pattern("*/proc/*/mountinfo") + unique: dict[tuple, dict] = {} + for file_path in mount_files: + pid_match = re.search(r"/proc/(\d+)/mountinfo$", file_path) + if not pid_match: + continue + entries = self.parse_mountinfo( + self._get_file_content(file_path).decode("utf-8", errors="replace"), + int(pid_match.group(1)), + ) + for entry in entries: + identity = ( + entry["major_minor"], + entry["root"], + entry["mount_point"], + entry["device"], + entry["filesystem_type"], + entry["mount_options"], + ) + if identity in unique: + unique[identity]["process_ids"].extend(entry["process_ids"]) + else: + unique[identity] = entry + self.results = list(unique.values()) + self.log.info("Extracted %d unique mount records", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/processes.py b/src/mvt/android/modules/bugreport/processes.py new file mode 100644 index 0000000..4f9b334 --- /dev/null +++ b/src/mvt/android/modules/bugreport/processes.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. + +from mvt.android.artifacts.processes import Processes as ProcessesArtifact + +from .base import BugReportModule + + +class Processes(ProcessesArtifact, BugReportModule): + """Extract the process and thread table from dumpstate.""" + + def run(self) -> None: + data = self._get_dumpstate_file() + if not data: + self.log.error("Unable to find dumpstate file") + return + section = self.extract_command_section( + data.decode("utf-8", errors="replace"), + "------ PROCESSES AND THREADS", + ) + self.parse(section) + self.log.info("Identified %d running process threads", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/settings.py b/src/mvt/android/modules/bugreport/settings.py new file mode 100644 index 0000000..20180ad --- /dev/null +++ b/src/mvt/android/modules/bugreport/settings.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. + +from mvt.android.artifacts.settings import Settings as SettingsArtifact + +from .base import BugReportModule + + +class Settings(SettingsArtifact, BugReportModule): + """Extract all SettingsProvider namespaces and users.""" + + def run(self) -> None: + data = self._get_dumpstate_file() + if not data: + self.log.error("Unable to find dumpstate file") + return + section = self.extract_dumpsys_section( + data.decode("utf-8", errors="replace"), "DUMP OF SERVICE settings:" + ) + self.parse(section) + count = sum(len(settings) for settings in self.results.values()) + self.log.info("Identified %d Android settings", count) diff --git a/src/mvt/android/modules/bugreport/tombstones.py b/src/mvt/android/modules/bugreport/tombstones.py index c4a7afb..4d121a1 100644 --- a/src/mvt/android/modules/bugreport/tombstones.py +++ b/src/mvt/android/modules/bugreport/tombstones.py @@ -43,21 +43,76 @@ class Tombstones(TombstoneCrashArtifact, BugReportModule): ) return - for tombstone_file in sorted(tombstone_files): - tombstone_filename = tombstone_file.split("/")[-1] - modification_time = self._get_file_modification_time(tombstone_file) - tombstone_data = self._get_file_content(tombstone_file) + grouped: dict[str, dict[str, str]] = {} + for file_path in tombstone_files: + file_name = file_path.rsplit("/", 1)[-1] + source = "protobuf" if file_name.endswith(".pb") else "text" + crash_id = file_name.removesuffix(".pb") + grouped.setdefault(crash_id, {})[source] = file_path - try: - if tombstone_file.endswith(".pb"): - self.parse_protobuf( - tombstone_filename, modification_time, tombstone_data + for crash_id, paths in sorted(grouped.items()): + parsed_sources: dict[str, dict] = {} + source_records: dict[str, dict] = {} + for source in ("text", "protobuf"): + file_path = paths.get(source) + if file_path is None: + continue + file_name = file_path.rsplit("/", 1)[-1] + file_timestamp = self._get_file_modification_time(file_path) + source_info = { + "file_name": file_name, + "file_timestamp": file_timestamp.isoformat(), + "parsed": False, + "error": None, + "record": None, + } + try: + data = self._get_file_content(file_path) + if source == "protobuf": + record = self.parse_protobuf_record( + file_name, file_timestamp, data + ) + else: + record = self.parse_text_record(file_name, file_timestamp, data) + source_info["parsed"] = True + source_info["record"] = record + source_records[source] = record + except Exception as exc: + source_info["error"] = str(exc) + self.log.error( + "Error parsing tombstone file %s: %s", file_path, exc ) - else: - self.parse(tombstone_filename, modification_time, tombstone_data) - except ValueError as e: - # Catch any exceptions raised during parsing or validation. - self.log.error(f"Error parsing tombstone file {tombstone_file}: {e}") + parsed_sources[source] = source_info + + if not source_records: + continue + preferred = source_records.get("protobuf") or source_records["text"] + canonical = dict(preferred) + text_record = source_records.get("text") + if text_record: + for key, value in text_record.items(): + if canonical.get(key) in (None, "", [], {}): + canonical[key] = value + + differences = {} + protobuf_record = source_records.get("protobuf") + if text_record and protobuf_record: + for key in text_record.keys() & protobuf_record.keys(): + if key in ("file_name", "file_timestamp"): + continue + if text_record[key] != protobuf_record[key]: + differences[key] = { + "text": text_record[key], + "protobuf": protobuf_record[key], + } + canonical.update( + { + "crash_id": crash_id, + "sources": parsed_sources, + "differences": differences, + } + ) + self.results.append(canonical) self.log.info( "Extracted a total of %d tombstone files", diff --git a/src/mvt/android/modules/intrusion_logs/__init__.py b/src/mvt/android/modules/intrusion_logs/__init__.py index f8be973..a0e7bdd 100644 --- a/src/mvt/android/modules/intrusion_logs/__init__.py +++ b/src/mvt/android/modules/intrusion_logs/__init__.py @@ -3,11 +3,13 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .connect_event import ConnectEvent from .dns_event import DnsEvent from .security_event import SecurityEvent -INTRUSION_LOGS_MODULES = [ +INTRUSION_LOGS_MODULES: list[type[MVTModule]] = [ DnsEvent, ConnectEvent, SecurityEvent, diff --git a/src/mvt/cli.py b/src/mvt/cli.py new file mode 100644 index 0000000..6e8c667 --- /dev/null +++ b/src/mvt/cli.py @@ -0,0 +1,110 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import click + +from mvt.common.cli_plugins import ( + MVT_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, + load_cli_commands_option, + register_cli_plugins, +) +from mvt.common.cmd_plugins import plugins +from mvt.common.completion import completion +from mvt.common.help import ( + HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, + HELP_MSG_DISABLE_UPDATE_CHECK, + HELP_MSG_STIX2, + HELP_MSG_VERBOSE, + HELP_MSG_VERSION, +) +from mvt.common.logo import logo +from mvt.common.updates import IndicatorsUpdates +from mvt.common.utils import init_logging, set_verbose_logging + +init_logging() + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +# ============================================================================== +# Main +# ============================================================================== +@click.group(invoke_without_command=True) +@load_cli_commands_option +@click.option( + "--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK +) +@click.option( + "--disable-indicator-update-check", + is_flag=True, + help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, +) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.pass_context +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): + """Mobile Verification Toolkit. + + The 'mvt-ios' and 'mvt-android' CLI commands are used to perform + forensic analysis on IOS and Android devices. + """ + ctx.ensure_object(dict) + ctx.obj["disable_version_check"] = disable_update_check + ctx.obj["disable_indicator_check"] = disable_indicator_update_check + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) + if ctx.invoked_subcommand != "completion": + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +# ============================================================================== +# Command: download-iocs +# ============================================================================== +@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) +def download_iocs(): + ioc_updates = IndicatorsUpdates() + ioc_updates.update() + + +# ============================================================================== +# Command: completion +# ============================================================================== +cli.add_command(completion) + + +# ============================================================================== +# Command: version +# ============================================================================== +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) +def version(): + return + + +# The plugins command is registered as a built-in command, before any external +# command, so that an installed package can never replace it. +cli.add_command(plugins) + + +# ============================================================================== +# Entry point of the mvt console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/common/cli_plugins.py b/src/mvt/common/cli_plugins.py index acf4c6d..17766ac 100644 --- a/src/mvt/common/cli_plugins.py +++ b/src/mvt/common/cli_plugins.py @@ -15,10 +15,20 @@ from typing import Iterable import click +# This module is imported by every CLI at start-up, and by shell completion on +# every keystroke, so it must stay cheap: nothing here may import the module +# machinery (mvt.common.module_loader and what it pulls in). + IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins" ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins" +# Commands in this group are registered on the platform-neutral mvt command only. +NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins" +MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS" MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS" MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS" +# Prefix of the import name given to a command file loaded from a path. Shared +# with module_loader, which recognises such files when naming their loggers. +CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_" log = logging.getLogger(__name__) @@ -54,7 +64,7 @@ class BrokenPluginCommand(click.Command): def _module_name_for_path(path: Path) -> str: digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] - return f"_mvt_custom_command_{path.stem}_{digest}" + return f"{CUSTOM_COMMAND_MODULE_PREFIX}{path.stem}_{digest}" def _iter_command_files(path: Path) -> Iterable[Path]: @@ -255,6 +265,16 @@ def register_cli_plugins( entry_point_group: str, environment_variable: str, ) -> None: + """Register the external commands of one CLI on its group. + + Each CLI has one entry-point group and one environment variable of its + own, so a command package chooses the CLIs its commands are added to. + + :param group: CLI group to register the external commands on. + :param entry_point_group: Entry-point group of the CLI. + :param environment_variable: Name of the environment variable holding a + path to load commands from. + """ environment_path = os.environ.get(environment_variable) if environment_path: register_cli_commands_from_path(group, environment_path) diff --git a/src/mvt/common/cmd_check_iocs.py b/src/mvt/common/cmd_check_iocs.py index c227659..c2dbdcd 100644 --- a/src/mvt/common/cmd_check_iocs.py +++ b/src/mvt/common/cmd_check_iocs.py @@ -9,6 +9,7 @@ from typing import Optional from mvt.common.command import Command from mvt.common.module import MVTModule +from mvt.common.module_loader import get_module_logger from mvt.common.utils import exec_or_profile log = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class CmdCheckIOCS(Command): ) m = iocs_module.from_json( - file_path, log=logging.getLogger(iocs_module.__module__) + file_path, log=get_module_logger(iocs_module) ) if not m: log.warning("No result from this module, skipping it") diff --git a/src/mvt/common/cmd_plugins.py b/src/mvt/common/cmd_plugins.py new file mode 100644 index 0000000..21619eb --- /dev/null +++ b/src/mvt/common/cmd_plugins.py @@ -0,0 +1,205 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import importlib.metadata +import logging +from typing import Optional + +import click +from rich.console import Console +from rich.table import Table + +from .cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from .config import settings +from .help import ( + HELP_MSG_PLUGINS, + HELP_MSG_PLUGINS_CHECK_UPDATES, + HELP_MSG_PLUGINS_LIST, +) +from .module import MVTModule +from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url +from .updates import ( + SHORT_COMMIT_LENGTH, + PluginUpdates, + installed_plugin_distributions, +) + +log = logging.getLogger(__name__) + +CLI_PLUGIN_GROUPS = ( + IOS_CLI_PLUGIN_GROUP, + ANDROID_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) + + +def _entry_points(group: str) -> list[importlib.metadata.EntryPoint]: + try: + return list(importlib.metadata.entry_points(group=group)) + except Exception as exc: + log.warning("Unable to discover the entry points in group %s: %s", group, exc) + return [] + + +def _entry_point_distribution( + entry_point: importlib.metadata.EntryPoint, +) -> Optional[str]: + dist = getattr(entry_point, "dist", None) + if dist is None: + return None + try: + return dist.name + except Exception: + return None + + +def _distribution_version(dist: importlib.metadata.Distribution) -> str: + try: + return dist.version or "unknown" + except Exception: + return "unknown" + + +def _distribution_origin(dist: importlib.metadata.Distribution) -> str: + """Describe where a plugin package was installed from.""" + direct_url = distribution_direct_url(dist) + if direct_url is None: + return "pypi" + + vcs_info = direct_url.get("vcs_info") + if isinstance(vcs_info, dict): + commit = vcs_info.get("commit_id") or "" + if commit: + return f"git+{commit[:SHORT_COMMIT_LENGTH]}" + return "git" + + return "local" + + +def _contributed_modules( + entry_points: list[importlib.metadata.EntryPoint], distribution: str +) -> str: + """Count the forensic modules a plugin package contributes. + + Entry points are resolved the way MVT resolves them when it loads + modules, but a broken entry point is reported instead of raising: listing + the installed plugins must work even when one of them is faulty. + """ + count = 0 + broken = False + + for entry_point in entry_points: + if _entry_point_distribution(entry_point) != distribution: + continue + try: + loaded = entry_point.load() + if callable(loaded) and not isinstance(loaded, type): + loaded = loaded() + count += sum( + 1 + for module in loaded + if isinstance(module, type) and issubclass(module, MVTModule) + ) + except (Exception, SystemExit) as exc: + log.debug( + "Unable to load the modules of entry point %s (%s): %s", + entry_point.name, + entry_point.value, + exc, + ) + broken = True + + if broken: + return f"{count} (error)" if count else "error" + + return str(count) + + +def _contributed_commands( + entry_points: list[importlib.metadata.EntryPoint], distribution: str +) -> str: + names = { + entry_point.name + for entry_point in entry_points + if _entry_point_distribution(entry_point) == distribution + } + + return ", ".join(sorted(names)) if names else "-" + + +@click.group("plugins", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS) +def plugins() -> None: + pass + + +@plugins.command("list", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS_LIST) +def list_plugins() -> None: + distributions = installed_plugin_distributions() + if not distributions: + click.echo("No MVT plugins are installed.") + return + + module_entry_points = _entry_points(MODULES_ENTRY_POINT_GROUP) + command_entry_points = [] + for group in CLI_PLUGIN_GROUPS: + command_entry_points.extend(_entry_points(group)) + + table = Table(title="Installed MVT plugins") + table.add_column("Name", style="bold") + table.add_column("Version") + table.add_column("Origin") + table.add_column("Modules", justify="right") + table.add_column("Commands") + + for dist in distributions: + name = dist.name + table.add_row( + name, + _distribution_version(dist), + _distribution_origin(dist), + _contributed_modules(module_entry_points, name), + _contributed_commands(command_entry_points, name), + ) + + Console().print(table) + + +@plugins.command( + "check-updates", + context_settings=CONTEXT_SETTINGS, + help=HELP_MSG_PLUGINS_CHECK_UPDATES, + short_help="Check the installed plugins for updates", +) +def check_plugin_updates() -> None: + if not settings.NETWORK_ACCESS_ALLOWED: + click.echo( + "Network access is disabled, cannot check for plugin updates. " + "Enable NETWORK_ACCESS_ALLOWED in the MVT configuration to check." + ) + return + + if not installed_plugin_distributions(): + click.echo("No MVT plugins are installed.") + return + + findings = PluginUpdates().check() + if not findings: + click.echo("All plugins are up to date.") + return + + click.echo("Plugin updates available:") + for finding in findings: + click.echo(f" {finding['name']} {finding['installed']} → {finding['latest']}") + click.echo(f" Upgrade with: {finding['upgrade_command']}") + + click.echo( + "\nMVT does not install plugin updates. Run the command above when you " + "decide to upgrade." + ) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index aab53e6..1242941 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -19,7 +19,12 @@ from .alerts import AlertLevel, AlertStore from .config import settings from .indicators import Indicators from .module import EncryptedBackupError, MVTModule, run_module, save_timeline -from .module_loader import module_supports_command +from .module_loader import ( + ModuleOrigin, + get_module_logger, + get_module_origin, + module_supports_command, +) from .module_types import ModuleTimeline, URLResult from .utils import ( CustomJSONEncoder, @@ -67,6 +72,10 @@ class Command: # down a password to decrypt a backup or flags which are need by some modules. self.module_options = module_options if module_options else {} + # This dictionary maps the modules which were replaced by a module + # declaring `replaces` to the module which took their place. + self.module_replacements: dict[type[MVTModule], type[MVTModule]] = {} + # This list will contain all executed modules. # We can use this to reference e.g. self.executed[0].results. self.executed: list[MVTModule] = [] @@ -210,10 +219,34 @@ class Command: for file in generate_hashes_from_path(self.target_path, self.log): self.hash_values.append(file) + @staticmethod + def _modules_by_origin( + modules: list[type[MVTModule]], + ) -> dict[ModuleOrigin, list[str]]: + origins: dict[ModuleOrigin, list[str]] = {} + for module in modules: + origins.setdefault(get_module_origin(module), []).append(module.__name__) + return origins + def list_modules(self) -> None: self.log.info("Following is the list of available %s modules:", self.name) - for module in self._available_modules(): - self.log.info(" - %s", module.__name__) + for origin, module_names in self._modules_by_origin( + self._available_modules() + ).items(): + self.log.info( + " - Modules from %s: %s", origin.label, ", ".join(module_names) + ) + + def _log_loaded_modules(self, modules: list[type[MVTModule]]) -> None: + """Record the loaded modules and their origins for auditability.""" + for origin, module_names in self._modules_by_origin(modules).items(): + self.log.info( + "Loaded %d %s modules from %s: %s", + len(module_names), + self.name, + origin.label, + ", ".join(module_names), + ) def _available_modules(self) -> list[type[MVTModule]]: modules = list(self.modules) @@ -228,7 +261,174 @@ class Command: if module not in deduplicated: deduplicated.append(module) - return deduplicated + available = self._apply_replacements(deduplicated) + self._warn_about_slug_collisions(available) + + return available + + def _warn_about_slug_collisions(self, modules: list[type[MVTModule]]) -> None: + """Report modules writing their results to the same file. + + Results are stored in a file named after the module slug, so two + modules sharing one silently overwrite each other. Replacements are + already resolved here, so a module deliberately taking over the slug + of the module it replaces is not reported: the module it replaced is + no longer part of the run. + """ + modules_by_slug: dict[str, type[MVTModule]] = {} + for module in modules: + slug = module.get_slug() + first = modules_by_slug.setdefault(slug, module) + if first is module: + continue + + self.log.warning( + "Modules %s from %s and %s from %s both use the slug %s. If " + "both run, whichever runs last overwrites the results of the " + "other in %s.json.", + first.__name__, + get_module_origin(first).label, + module.__name__, + get_module_origin(module).label, + slug, + slug, + ) + + def _declared_replacements( + self, modules: list[type[MVTModule]] + ) -> dict[type[MVTModule], type[MVTModule]]: + """Return the replacements declared by the given modules.""" + replacements: dict[type[MVTModule], type[MVTModule]] = {} + for module in modules: + replaced = module.replaces + if replaced is None or replaced is module: + continue + + if not module.enabled: + # Replacing a module must not disable it, as a disabled + # replacement never runs in its place. + self.log.debug( + "Module %s is disabled and does not replace module %s.", + module.__name__, + replaced.__name__, + ) + continue + + if replaced not in modules: + # A module can support several commands while the module it + # replaces is only available in some of them. + self.log.debug( + "Module %s replaces module %s, which is not available " + "for the %s command.", + module.__name__, + replaced.__name__, + self.name, + ) + continue + + if replaced in replacements: + self.log.warning( + "Modules %s and %s both replace module %s. Both of them " + "will run, %s will not, and modules depending on %s will " + "use the results of %s. Replacements which share the slug " + "of %s overwrite each other's results file.", + replacements[replaced].__name__, + module.__name__, + replaced.__name__, + replaced.__name__, + replaced.__name__, + replacements[replaced].__name__, + replaced.__name__, + ) + continue + + replacements[replaced] = module + + return replacements + + def _drop_replacement_cycles( + self, replacements: dict[type[MVTModule], type[MVTModule]] + ) -> None: + """Undo the replacements between modules which replace each other.""" + cyclic: set[type[MVTModule]] = set() + for replaced in replacements: + walked: list[type[MVTModule]] = [] + module = replaced + while module in replacements and module not in cyclic: + if module in walked: + cyclic.update(walked[walked.index(module) :]) + break + walked.append(module) + module = replacements[module] + + if not cyclic: + return + + self.log.warning( + "Modules %s replace each other in a cycle. None of them replaces " + "anything and all of them will run.", + ", ".join(sorted(module.__name__ for module in cyclic)), + ) + for module in cyclic: + replacements.pop(module, None) + + def _log_replacement( + self, + replaced: type[MVTModule], + module: type[MVTModule], + replacement: type[MVTModule], + ) -> None: + """Report an applied replacement, and any problem with it.""" + if not issubclass(module, replaced): + self.log.warning( + "Module %s replaces module %s but is not a subclass of it. " + "Its results might not be compatible with what modules " + "depending on %s expect.", + module.__name__, + replaced.__name__, + replaced.__name__, + ) + + if replaced in module.dependencies: + self.log.warning( + "Module %s depends on module %s, which it also replaces. The " + "dependency cannot be satisfied: a replacement has to produce " + "that data itself.", + module.__name__, + replaced.__name__, + ) + + self.log.info( + "Module %s from %s replaces module %s from %s.", + replacement.__name__, + get_module_origin(replacement).label, + replaced.__name__, + get_module_origin(replaced).label, + ) + + def _apply_replacements( + self, modules: list[type[MVTModule]] + ) -> list[type[MVTModule]]: + """Drop the modules superseded by a module declaring `replaces`.""" + declared = self._declared_replacements(modules) + self._drop_replacement_cycles(declared) + + replacements: dict[type[MVTModule], type[MVTModule]] = {} + for replaced, module in declared.items(): + # Follow chains of replacements, so that a dependency on a + # replaced module always resolves to a module which is part of + # the run. + replacement = module + while replacement in declared: + replacement = declared[replacement] + + # Only the replacements which are applied are reported, so that + # the record matches the modules which actually run. + self._log_replacement(replaced, module, replacement) + replacements[replaced] = replacement + + self.module_replacements = replacements + return [module for module in modules if module not in replacements] def init(self) -> None: raise NotImplementedError @@ -288,42 +488,196 @@ class Command: console.print("") console.print(panel) + def _module_dependencies( + self, module: type[MVTModule] + ) -> list[tuple[type[MVTModule], type[MVTModule]]]: + """Return the (declared, resolved) dependencies of a module. + + A dependency on a module which was replaced is resolved to the module + which took its place. A module which replaces one of its own + dependencies is not made to depend on itself, while a module which + declares itself as a dependency is left alone and still fails the + circular dependency check. + """ + dependencies = [] + for dependency in module.dependencies: + resolved = self.module_replacements.get(dependency, dependency) + if resolved is module and dependency is not module: + continue + dependencies.append((dependency, resolved)) + + return dependencies + + @staticmethod + def _dependency_name( + declared: type[MVTModule], resolved: type[MVTModule] + ) -> str: + """Return how a dependency is named in the messages about it. + + A dependency is named as the module which declared it wrote it, and, + when that module was replaced, as the module which runs in its place. + """ + if declared is resolved: + return declared.__name__ + + return f"{resolved.__name__} (replacing module {declared.__name__})" + + def _selected_modules( + self, modules: list[type[MVTModule]] + ) -> Optional[list[type[MVTModule]]]: + """Return the modules explicitly requested, or all the enabled ones. + + Returns None when a module was requested by name and no module of + that name can be run. + """ + if not self.module_name: + return [module for module in modules if module.enabled] + + selected = [ + module for module in modules if module.__name__ == self.module_name + ] + + # A module replacing another one does not have to keep its name, so + # the name of a replaced module selects its replacement. + if not selected: + for replaced, replacement in self.module_replacements.items(): + if replaced.__name__ != self.module_name or replacement in selected: + continue + self.log.info( + "Module %s was replaced by module %s, which is run " + "in its place.", + replaced.__name__, + replacement.__name__, + ) + selected.append(replacement) + + if not selected: + self.log.warning( + "No module named %s is available for the %s command. " + "No modules will be run.", + self.module_name, + self.name, + ) + return None + + return selected + + def _skipped_modules( + self, + required: list[type[MVTModule]], + module_indexes: dict[type[MVTModule], int], + ) -> dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]]: + """Return the modules to drop because a dependency is unavailable. + + A module declaring a dependency this command cannot provide is unable + to run, and so is every module depending on it. Dropping only those + keeps a single wrong declaration - in a module scoped to several + commands, for example - from silencing an entire analysis. + + The returned mapping gives, for each skipped module, the module which + is missing a dependency and the dependency it is missing. + """ + skipped: dict[type[MVTModule], tuple[type[MVTModule], type[MVTModule]]] = {} + # Skipping the one module a run was asked for leaves nothing to run, + # which the caller reports instead. + remainder = ( + "" if self.module_name else " The rest of the analysis will still run." + ) + + # Skipping one module can skip the modules depending on it, which the + # pass over the module list may already have gone past, so repeat the + # pass until nothing changes. + changed = True + while changed: + changed = False + for module in required: + if module in skipped: + continue + + for declared, dependency in self._module_dependencies(module): + if dependency not in module_indexes: + # A replaced dependency always resolves to a module of + # this command, so an unavailable one is always the + # module class the author declared. + skipped[module] = (module, declared) + changed = True + self.log.warning( + "Module %s will be SKIPPED: it depends on module " + "%s, which is not available in this command.%s", + module.__name__, + declared.__name__, + remainder, + ) + break + + if dependency in skipped: + root, missing = skipped[dependency] + skipped[module] = (root, missing) + changed = True + if dependency is root: + self.log.warning( + "Module %s will be SKIPPED: it depends on " + "module %s, itself skipped for depending on " + "unavailable module %s.%s", + module.__name__, + self._dependency_name(declared, dependency), + missing.__name__, + remainder, + ) + else: + self.log.warning( + "Module %s will be SKIPPED: it depends on " + "skipped module %s, in a chain starting at " + "module %s, which depends on unavailable " + "module %s.%s", + module.__name__, + self._dependency_name(declared, dependency), + root.__name__, + missing.__name__, + remainder, + ) + break + + return skipped + def _ordered_modules(self) -> Optional[list[type[MVTModule]]]: """Return enabled modules in stable topological order.""" modules = self._available_modules() module_indexes = {module: index for index, module in enumerate(modules)} - if self.module_name: - selected = [ - module for module in modules if module.__name__ == self.module_name - ] - else: - selected = [module for module in modules if module.enabled] + selected = self._selected_modules(modules) + if selected is None: + return None - required = set(selected) + required: set[type[MVTModule]] = set() pending = list(selected) while pending: module = pending.pop() - for dependency in module.dependencies: - if dependency not in module_indexes: - self.log.warning( - "Module %s depends on unavailable module %s. " - "No modules will be run.", - module.__name__, - dependency.__name__, - ) - return None - if dependency not in required: - required.add(dependency) + if module in required: + continue + required.add(module) + for _, dependency in self._module_dependencies(module): + # Unavailable dependencies are reported by _skipped_modules(). + if dependency in module_indexes: pending.append(dependency) + ordered_required = sorted(required, key=lambda module: module_indexes[module]) + skipped = self._skipped_modules(ordered_required, module_indexes) + runnable = [module for module in ordered_required if module not in skipped] + if skipped and not runnable: + self.log.warning( + "Every selected module was skipped for an unavailable " + "dependency. No modules will be run." + ) + return None + dependents: dict[type[MVTModule], list[type[MVTModule]]] = { - module: [] for module in required + module: [] for module in runnable } - indegree = {module: 0 for module in required} - for module in required: - for dependency in module.dependencies: - if dependency not in required: + indegree = {module: 0 for module in runnable} + for module in runnable: + for _, dependency in self._module_dependencies(module): + if dependency not in indegree: continue dependents[dependency].append(module) indegree[module] += 1 @@ -342,7 +696,7 @@ class Command: if indegree[dependent] == 0: heappush(ready, (module_indexes[dependent], dependent)) - if len(ordered) != len(required): + if len(ordered) != len(runnable): cyclic_modules = sorted( (module.__name__ for module, count in indegree.items() if count > 0) ) @@ -360,6 +714,8 @@ class Command: if ordered_modules is None: return + self._log_loaded_modules(ordered_modules) + try: self.init() except NotImplementedError: @@ -368,7 +724,7 @@ class Command: executed_by_type: dict[type[MVTModule], MVTModule] = {} for module in ordered_modules: - module_logger = logging.getLogger(module.__module__) + module_logger = get_module_logger(module) m = module( target_path=self.target_path, @@ -376,9 +732,12 @@ class Command: module_options=self.module_options, log=module_logger, ) + # Dependencies are keyed by the module class they declare, even + # when it was replaced, so that a module asking for the results + # of a replaced module receives those of its replacement. m.dependency_modules = { - dependency: executed_by_type[dependency] - for dependency in module.dependencies + dependency: executed_by_type[resolved] + for dependency, resolved in self._module_dependencies(module) } if self.iocs.total_ioc_count: diff --git a/src/mvt/common/completion.py b/src/mvt/common/completion.py index 6466a6d..0b72b4e 100644 --- a/src/mvt/common/completion.py +++ b/src/mvt/common/completion.py @@ -9,34 +9,83 @@ import shlex import click from click.shell_completion import get_completion_class +from .help import HELP_MSG_COMPLETION + SUPPORTED_SHELLS = ("bash", "zsh", "fish") +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +COMPLETION_INSTRUCTIONS = """Shell completion for mvt, mvt-ios and mvt-android -def completion_instructions(program_name: str) -> str: - return f"""Shell completion for {program_name} - -Print a completion script: - {program_name} completion bash > ~/.{program_name}-complete.bash - {program_name} completion zsh > ~/.{program_name}-complete.zsh - mkdir -p ~/.config/fish/completions - {program_name} completion fish > ~/.config/fish/completions/{program_name}.fish +Print one completion script covering the three commands: + mvt completion bash > ~/.mvt-complete.bash + mvt completion zsh > ~/.mvt-complete.zsh + mkdir -p ~/.config/fish/conf.d + mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish Load the generated Bash script from ~/.bashrc: - [ -f ~/.{program_name}-complete.bash ] && . ~/.{program_name}-complete.bash + [ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash Load the generated Zsh script from ~/.zshrc: - [ -f ~/.{program_name}-complete.zsh ] && . ~/.{program_name}-complete.zsh + [ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh -Fish loads completion files from ~/.config/fish/completions automatically. +Fish loads the files in ~/.config/fish/conf.d automatically. -To write these files and update Bash/Zsh shell configuration automatically: - {program_name} completion bash --install - {program_name} completion zsh --install - {program_name} completion fish --install +To write these files and update the Bash/Zsh shell configuration automatically: + mvt completion bash --install + mvt completion zsh --install + mvt completion fish --install """ +def _mvt_programs() -> list[tuple[str, click.Command]]: + """Return the console script name and CLI group of every MVT program. + + The three CLIs are imported here rather than at module level: mvt.cli + imports this module while it is being defined, and generating a completion + script should not make the start-up of `mvt` import the platform CLIs. + """ + from mvt.android.cli import cli as android_cli + from mvt.cli import cli as mvt_cli + from mvt.ios.cli import cli as ios_cli + + return [("mvt", mvt_cli), ("mvt-ios", ios_cli), ("mvt-android", android_cli)] + + +@click.command( + "completion", + context_settings=CONTEXT_SETTINGS, + help=HELP_MSG_COMPLETION, + short_help="Generate or install shell completion", +) +@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) +@click.option( + "--install", + is_flag=True, + help="Write completion files and update shell configuration.", +) +def completion(shell, install): + if shell is None: + if install: + raise click.UsageError("A shell is required when using --install.") + click.echo(COMPLETION_INSTRUCTIONS) + return + + if install: + script_path = install_completion_script(shell) + click.echo( + f"Installed {shell} completion for mvt, mvt-ios and mvt-android " + f"to {script_path}" + ) + if shell in ("bash", "zsh"): + click.echo(f"Updated ~/.{shell}rc") + else: + click.echo("Fish loads the files in ~/.config/fish/conf.d automatically.") + return + + click.echo(generate_mvt_completion_script(shell)) + + def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str: completion_class = get_completion_class(shell) if completion_class is None: @@ -46,41 +95,49 @@ def generate_completion_script(cli: click.Command, program_name: str, shell: str return completion_class(cli, {}, program_name, complete_var).source() -def install_completion_script( - cli: click.Command, - program_name: str, - shell: str, -) -> Path: - script = generate_completion_script(cli, program_name, shell) - script_path = _completion_script_path(program_name, shell) +def generate_mvt_completion_script(shell: str) -> str: + """Return one script completing every MVT command. + + Click names the completion function of each program after the program, so + the scripts of the three commands can simply be concatenated. + """ + scripts = [ + generate_completion_script(cli, program_name, shell).strip("\n") + for program_name, cli in _mvt_programs() + ] + return "\n\n".join(scripts) + + +def install_completion_script(shell: str) -> Path: + script = generate_mvt_completion_script(shell) + script_path = _completion_script_path(shell) script_path.parent.mkdir(parents=True, exist_ok=True) - script_path.write_text(script, encoding="utf-8") + script_path.write_text(f"{script}\n", encoding="utf-8") if shell in ("bash", "zsh"): - _install_shell_source_line(program_name, shell, script_path) + _install_shell_source_line(shell, script_path) return script_path -def _completion_script_path(program_name: str, shell: str) -> Path: +def _completion_script_path(shell: str) -> Path: home = Path.home() if shell == "fish": - return home / ".config" / "fish" / "completions" / f"{program_name}.fish" + # conf.d is sourced when the shell starts, unlike the completions + # folder, whose files fish loads on demand by command name. + return home / ".config" / "fish" / "conf.d" / "mvt-completion.fish" - return home / f".{program_name}-complete.{shell}" + return home / f".mvt-complete.{shell}" -def _install_shell_source_line(program_name: str, shell: str, script_path: Path) -> None: +def _install_shell_source_line(shell: str, script_path: Path) -> None: shell_config_path = Path.home() / f".{shell}rc" source_line = ( f"[ -f {shlex.quote(str(script_path))} ] && " f". {shlex.quote(str(script_path))}" ) - block = ( - f"# MVT shell completion for {program_name}\n" - f"{source_line}\n" - ) + block = f"# MVT shell completion\n{source_line}\n" if shell_config_path.exists(): shell_config = shell_config_path.read_text(encoding="utf-8") diff --git a/src/mvt/common/help.py b/src/mvt/common/help.py index 7461538..9d65f7b 100644 --- a/src/mvt/common/help.py +++ b/src/mvt/common/help.py @@ -8,7 +8,7 @@ HELP_MSG_VERSION = "Show the currently installed version of MVT" HELP_MSG_OUTPUT = "Specify a path to a folder where you want to store JSON results" HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)" HELP_MSG_FAST = "Avoid running time/resource consuming features" -HELP_MSG_LIST_MODULES = "Print list of available modules and exit" +HELP_MSG_LIST_MODULES = "Print list of available modules and their source, then exit" HELP_MSG_MODULE = "Name of a single module you would like to run instead of all" HELP_MSG_LOAD_MODULE = ( "Load custom MVT module(s) from a Python file or folder " @@ -17,11 +17,22 @@ HELP_MSG_LOAD_MODULE = ( HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing" HELP_MSG_HASHES = "Generate hashes of all the files analyzed" HELP_MSG_VERBOSE = "Verbose mode" +HELP_MSG_VERBOSE_COMMAND = ( + "Verbose mode (kept for compatibility, pass --verbose before the command " + "name instead)" +) HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators" HELP_MSG_STIX2 = "Download public STIX2 indicators" HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check" HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check" -HELP_MSG_COMPLETION = "Generate or install shell completion" +HELP_MSG_COMPLETION = ( + "Generate or install shell completion for mvt, mvt-ios and mvt-android" +) +HELP_MSG_PLUGINS = "Inspect the installed MVT plugin packages" +HELP_MSG_PLUGINS_LIST = "List the installed plugins and what they contribute to MVT" +HELP_MSG_PLUGINS_CHECK_UPDATES = ( + "Check the installed plugins for updates without installing them" +) # IOS Specific HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup" diff --git a/src/mvt/common/logo.py b/src/mvt/common/logo.py index 048ee22..96836a9 100644 --- a/src/mvt/common/logo.py +++ b/src/mvt/common/logo.py @@ -8,10 +8,125 @@ import logging import requests from rich import print as rich_print -from .updates import IndicatorsUpdates, MVTUpdates +from .config import settings +from .updates import ( + IndicatorsUpdates, + MVTUpdates, + PluginUpdates, + installed_plugin_distributions, +) from .version import MVT_VERSION +def _check_version_updates(log: logging.Logger) -> None: + try: + mvt_updates = MVTUpdates() + latest_version = mvt_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for MVT updates.[/bold] " + "You may be working offline. Please update MVT regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check latest MVT version: %s", e) + else: + if latest_version: + rich_print( + f"\t[bold]Version {latest_version} is available! " + "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" + ) + + +def _check_indicator_updates(log: logging.Logger) -> None: + ioc_updates = IndicatorsUpdates() + + # Before proceeding, we check if we have downloaded an indicators index. + # If not, there's no point in proceeding with the updates check. + if ioc_updates.get_latest_update() == 0: + rich_print( + "\t[bold]You have not yet downloaded any indicators, check " + "the `download-iocs` command![/bold]" + ) + return + + # We only perform this check at a fixed frequency, in order to not + # overburden the user with too many lookups if the command is being run + # multiple times. + should_check, hours = ioc_updates.should_check() + if not should_check: + rich_print( + f"\tIndicators updates checked recently, next automatic check " + f"in {int(hours)} hours" + ) + return + + try: + ioc_to_update = ioc_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for indicator updates.[/bold] " + "You may be working offline. Please update MVT indicators regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check latest MVT indicators: %s", e) + else: + if ioc_to_update: + rich_print( + "\t[bold]There are updates to your indicators files! " + "Run the `download-iocs` command to update![/bold]" + ) + else: + rich_print("\tYour indicators files seem to be up to date.") + + +def _print_plugin_updates(findings: list) -> None: + if not findings: + return + + rich_print("\t[bold]Plugin updates available:[/bold]") + for finding in findings: + rich_print( + f"\t {finding['name']} {finding['installed']} → " + f"{finding['latest']} ({finding['upgrade_command']})" + ) + + +def _check_plugin_updates(log: logging.Logger) -> None: + if not settings.NETWORK_ACCESS_ALLOWED: + return + + # This runs on every command, so nothing here, including reading back what + # the latest check stored, may ever interrupt MVT. + try: + distributions = installed_plugin_distributions() + + # There is nothing to check when MVT was not extended with any plugin. + if not distributions: + return + + plugin_updates = PluginUpdates() + + # We only perform this check at a fixed frequency, in order to not + # overburden the user (and the plugin repositories) with too many + # lookups. In between checks we print the findings of the latest one, + # leaving out those which no longer apply to what is installed. + should_check, _ = plugin_updates.should_check() + if not should_check: + _print_plugin_updates(plugin_updates.current_findings(distributions)) + return + + findings = plugin_updates.check() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + rich_print( + "\t[bold]Note: Could not check for plugin updates.[/bold] " + "You may be working offline. Please update your MVT plugins regularly." + ) + except Exception as e: + log.error("Error encountered when trying to check MVT plugin updates: %s", e) + else: + _print_plugin_updates(findings) + + def check_updates( disable_version_check: bool = False, disable_indicator_check: bool = False ) -> None: @@ -19,77 +134,25 @@ def check_updates( # First we check for MVT version updates. if not disable_version_check: - try: - mvt_updates = MVTUpdates() - latest_version = mvt_updates.check() - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): - rich_print( - "\t\t[bold]Note: Could not check for MVT updates.[/bold] " - "You may be working offline. Please update MVT regularly." - ) - except Exception as e: - log.error( - "Error encountered when trying to check latest MVT version: %s", e - ) - else: - if latest_version: - rich_print( - f"\t\t[bold]Version {latest_version} is available! " - "Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]" - ) + _check_version_updates(log) # Then we check for indicators files updates. if not disable_indicator_check: - ioc_updates = IndicatorsUpdates() + _check_indicator_updates(log) - # Before proceeding, we check if we have downloaded an indicators index. - # If not, there's no point in proceeding with the updates check. - if ioc_updates.get_latest_update() == 0: - rich_print( - "\t\t[bold]You have not yet downloaded any indicators, check " - "the `download-iocs` command![/bold]" - ) - return - - # We only perform this check at a fixed frequency, in order to not - # overburden the user with too many lookups if the command is being run - # multiple times. - should_check, hours = ioc_updates.should_check() - if not should_check: - rich_print( - f"\t\tIndicators updates checked recently, next automatic check " - f"in {int(hours)} hours" - ) - return - - try: - ioc_to_update = ioc_updates.check() - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): - rich_print( - "\t\t[bold]Note: Could not check for indicator updates.[/bold] " - "You may be working offline. Please update MVT indicators regularly." - ) - except Exception as e: - log.error( - "Error encountered when trying to check latest MVT indicators: %s", e - ) - else: - if ioc_to_update: - rich_print( - "\t\t[bold]There are updates to your indicators files! " - "Run the `download-iocs` command to update![/bold]" - ) - else: - rich_print("\t\tYour indicators files seem to be up to date.") + # Finally we check for updates to the installed plugin packages. MVT never + # installs an update itself, it only reports the command which does. + if not disable_version_check: + _check_plugin_updates(log) def logo( disable_version_check: bool = False, disable_indicator_check: bool = False ) -> None: rich_print("\n") - rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit") - rich_print("\t\thttps://mvt.re") - rich_print(f"\t\tVersion: {MVT_VERSION}") + rich_print("\t[bold]MVT - Mobile Verification Toolkit[/bold]\n") + rich_print("\thttps://mvt.re") + rich_print(f"\tVersion: {MVT_VERSION}\n") check_updates(disable_version_check, disable_indicator_check) diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index cd127a6..1257aa6 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -46,6 +46,11 @@ class MVTModule: slug: Optional[str] = None dependencies: Sequence[type["MVTModule"]] = () supported_commands: Sequence[tuple[str, str]] = () + # A custom module can name a module class it supersedes, usually a + # built-in one. When both are available to a command, the named module is + # dropped from the run and this module takes its place, including in the + # dependencies of any other module. + replaces: Optional[type["MVTModule"]] = None def __init__( self, diff --git a/src/mvt/common/module_loader.py b/src/mvt/common/module_loader.py index da94826..51b3836 100644 --- a/src/mvt/common/module_loader.py +++ b/src/mvt/common/module_loader.py @@ -4,18 +4,31 @@ # https://license.mvt.re/1.1/ import hashlib +import importlib.metadata import importlib.util import inspect +import json import logging import os +import re import sys +from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from types import ModuleType from typing import Iterable, Optional +from .cli_plugins import CUSTOM_COMMAND_MODULE_PREFIX from .module import MVTModule +from .version import MVT_VERSION MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES" +MODULES_ENTRY_POINT_GROUP = "mvt.modules" +EXTERNAL_LOGGER_NAMESPACE = "mvt.ext" +PLUGIN_PACKAGE_PREFIX = "mvt_plugin_" +_ORIGIN_ATTRIBUTE = "_mvt_module_origin" +_PATH_MODULE_PREFIX = "_mvt_custom_module_" +_LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$") log = logging.getLogger(__name__) @@ -23,15 +36,109 @@ class CustomModuleLoadError(Exception): pass +@dataclass(frozen=True) +class ModuleOrigin: + """Describes where a loaded module came from, for auditability. + + ``kind`` is one of ``builtin`` (shipped with MVT), ``package`` (loaded + from an installed package) or ``path`` (loaded from a file passed with + ``--load-module`` or the environment variable). + """ + + kind: str + name: str + version: Optional[str] = None + commit: Optional[str] = None + file_sha256: Optional[str] = None + + @property + def label(self) -> str: + label = self.name + if self.version: + label += f"@{self.version}" + label = f"'{label}'" + if self.commit: + label += f" (commit {self.commit})" + if self.file_sha256: + label += f" (sha256: {self.file_sha256})" + return label + + def _module_name_for_path(path: Path) -> str: digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16] - return f"_mvt_custom_module_{path.stem}_{digest}" + return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}" + + +def _is_builtin_logger_name(name: str) -> bool: + return name == "mvt" or name.startswith("mvt.") + + +def _loaded_file_stem(name: str, prefix: str) -> str: + """Recover a loaded file's name from the import name MVT gave it.""" + return _LOADED_FILE_DIGEST.sub("", name[len(prefix) :]) + + +def _external_logger_name(name: str) -> str: + """Return the "mvt.ext" logger name external code logs under.""" + top_level, separator, rest = name.partition(".") + if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len( + PLUGIN_PACKAGE_PREFIX + ): + name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest + return f"{EXTERNAL_LOGGER_NAMESPACE}.{name}" + + +def get_module_logger(module_class: type[MVTModule]) -> logging.Logger: + """Return the logger a module's records should be emitted through. + + Modules loaded from installed packages or file paths live outside the + "mvt" logger hierarchy, so their records would never reach the handlers + attached to the "mvt" logger and instead fall through to + logging.lastResort (which prints bare messages and drops anything below + WARNING). Their loggers are parented under the "mvt.ext" namespace, + keeping external module names from colliding with MVT's own logger + tree. File-path modules are named after their file instead of the + mangled internal import name, and packages following the recommended + "mvt_plugin_" naming convention log under "mvt.ext.". + """ + name = module_class.__module__ + if _is_builtin_logger_name(name): + return logging.getLogger(name) + + if name.startswith(_PATH_MODULE_PREFIX): + file_name = Path(get_module_origin(module_class).name).stem + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{file_name}") + + return logging.getLogger(_external_logger_name(name)) + + +def get_plugin_logger(name: str) -> logging.Logger: + """Return a general logger for use in custom MVT plugins. + + Call it with ``__name__``. The logger sits under "mvt.ext". That is + where get_module_logger() puts module classes. A file loaded with + --load-module or --load-command is named after the file. + """ + if _is_builtin_logger_name(name): + return logging.getLogger(name) + + # A file loaded with --load-module or --load-command is imported under a + # mangled name. Log it under the file it came from. get_module_logger() + # does the same for the module classes such a file defines. + for prefix in (_PATH_MODULE_PREFIX, CUSTOM_COMMAND_MODULE_PREFIX): + if name.startswith(prefix): + stem = _loaded_file_stem(name, prefix) + return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{stem}") + + return logging.getLogger(_external_logger_name(name)) def _iter_module_files(path: Path) -> Iterable[Path]: if path.is_file(): if path.suffix != ".py": - raise CustomModuleLoadError(f"Custom module file is not a Python file: {path}") + raise CustomModuleLoadError( + f"Custom module file is not a Python file: {path}" + ) yield path return @@ -59,7 +166,9 @@ def _load_python_file(path: Path) -> ModuleType: try: spec.loader.exec_module(module) except Exception as exc: - raise CustomModuleLoadError(f"Unable to import custom module {path}: {exc}") from exc + raise CustomModuleLoadError( + f"Unable to import custom module {path}: {exc}" + ) from exc return module @@ -84,17 +193,171 @@ def load_custom_modules_from_path(path: str) -> list[type[MVTModule]]: resolved_path = Path(path).expanduser().resolve() for module_file in _iter_module_files(resolved_path): + file_sha256 = hashlib.sha256(module_file.read_bytes()).hexdigest() loaded_module = _load_python_file(module_file) + origin = ModuleOrigin( + kind="path", name=str(module_file), file_sha256=file_sha256 + ) for module_class in discover_mvt_modules(loaded_module): key = (str(module_file), module_class.__qualname__) if key in seen: continue seen.add(key) + setattr(module_class, _ORIGIN_ATTRIBUTE, origin) custom_modules.append(module_class) return custom_modules +def _module_key(module_class: type[MVTModule]) -> tuple[str, str]: + try: + source = str(Path(inspect.getfile(module_class)).resolve()) + except (OSError, TypeError): + source = module_class.__module__ + return (source, module_class.__qualname__) + + +def distribution_direct_url(dist: importlib.metadata.Distribution) -> Optional[dict]: + """Return the PEP 610 direct URL metadata of a distribution, if recorded. + + Packages installed from an index have no direct URL metadata, while + packages installed directly from a repository or from a local folder + record where they were installed from in ``direct_url.json``. + """ + try: + direct_url_text = dist.read_text("direct_url.json") + if not direct_url_text: + return None + direct_url = json.loads(direct_url_text) + return direct_url if isinstance(direct_url, dict) else None + except Exception: + return None + + +def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]: + """Return the VCS commit a distribution was installed from, if recorded. + + Packages installed directly from a repository (``pip install git+...``) + record the commit in ``direct_url.json`` (PEP 610). + """ + vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info") + if not isinstance(vcs_info, dict): + return None + + commit = vcs_info.get("commit_id") + return commit if isinstance(commit, str) else None + + +def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin: + name = entry_point.name + version = None + commit = None + # Manually constructed entry points have no associated distribution. + dist = getattr(entry_point, "dist", None) + if dist is not None: + try: + name = dist.name or name + version = dist.version + except Exception: + pass + commit = _distribution_commit(dist) + return ModuleOrigin(kind="package", name=name, version=version, commit=commit) + + +@lru_cache(maxsize=1) +def _packages_distributions() -> dict[str, list[str]]: + try: + return dict(importlib.metadata.packages_distributions()) + except Exception: + return {} + + +def get_module_origin(module_class: type[MVTModule]) -> ModuleOrigin: + """Return the origin of a module class for auditing purposes.""" + origin = module_class.__dict__.get(_ORIGIN_ATTRIBUTE) + if isinstance(origin, ModuleOrigin): + return origin + + top_level = module_class.__module__.partition(".")[0] + if top_level == "mvt": + return ModuleOrigin(kind="builtin", name="mvt", version=MVT_VERSION) + + distributions = _packages_distributions().get(top_level) + if distributions: + name = distributions[0] + version = None + commit = None + try: + dist = importlib.metadata.distribution(name) + version = dist.version + commit = _distribution_commit(dist) + except Exception: + pass + return ModuleOrigin(kind="package", name=name, version=version, commit=commit) + + try: + source = str(Path(inspect.getfile(module_class)).resolve()) + except (OSError, TypeError): + source = module_class.__module__ + return ModuleOrigin(kind="path", name=source) + + +def load_installed_modules() -> list[type[MVTModule]]: + """Load MVT modules registered by installed packages. + + Packages register modules in the ``mvt.modules`` entry-point group. Each + entry point must resolve to an iterable of MVTModule subclasses, or to a + callable which returns one. A broken entry point is skipped with a + warning so that a faulty plugin package cannot break MVT. + """ + try: + entry_points = importlib.metadata.entry_points(group=MODULES_ENTRY_POINT_GROUP) + except Exception as exc: + log.warning( + "Unable to discover installed module packages in entry-point group %s: %s", + MODULES_ENTRY_POINT_GROUP, + exc, + ) + return [] + + installed_modules: list[type[MVTModule]] = [] + ordered_entry_points = sorted( + entry_points, key=lambda entry_point: (entry_point.name, entry_point.value) + ) + for entry_point in ordered_entry_points: + try: + loaded = entry_point.load() + if callable(loaded) and not isinstance(loaded, type): + loaded = loaded() + module_classes = list(loaded) + except (Exception, SystemExit) as exc: + log.warning( + "Unable to load modules from entry point %s (%s): %s", + entry_point.name, + entry_point.value, + exc, + ) + continue + + origin = _entry_point_origin(entry_point) + for module_class in module_classes: + if not ( + isinstance(module_class, type) and issubclass(module_class, MVTModule) + ): + log.warning( + "Entry point %s (%s) provided %r which is not an " + "MVTModule subclass", + entry_point.name, + entry_point.value, + module_class, + ) + continue + setattr(module_class, _ORIGIN_ATTRIBUTE, origin) + installed_modules.append(module_class) + + return installed_modules + + def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTModule]]: search_paths: list[str] = [] env_path = os.environ.get(MVT_CUSTOM_MODULES_ENV) @@ -105,10 +368,17 @@ def load_custom_modules(paths: Optional[Iterable[str]] = None) -> list[type[MVTM custom_modules: list[type[MVTModule]] = [] seen: set[tuple[str, str]] = set() + + for module_class in load_installed_modules(): + key = _module_key(module_class) + if key in seen: + continue + seen.add(key) + custom_modules.append(module_class) + for path in search_paths: for module_class in load_custom_modules_from_path(path): - source = Path(inspect.getfile(module_class)).resolve() - key = (str(source), module_class.__qualname__) + key = _module_key(module_class) if key in seen: continue seen.add(key) @@ -131,4 +401,14 @@ def module_supports_command( ) return False - return (platform, command) in {tuple(entry) for entry in supported_commands} + pairs = {tuple(entry) for entry in supported_commands} + if (platform, command) in pairs: + return True + + # A module which implements check_indicators() is re-checked by check-iocs + # for its platform. It does not need to declare the check-iocs pair. + return ( + command == "check-iocs" + and platform in {entry[0] for entry in pairs if entry} + and module_class.check_indicators is not MVTModule.check_indicators + ) diff --git a/src/mvt/common/plugin_config.py b/src/mvt/common/plugin_config.py new file mode 100644 index 0000000..f5e2099 --- /dev/null +++ b/src/mvt/common/plugin_config.py @@ -0,0 +1,299 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import contextlib +import json +import os +import re +import tempfile +from typing import Any, ClassVar, Dict, List, Tuple, Type, TypeVar + +import yaml +from appdirs import user_config_dir, user_data_dir +from pydantic import ValidationError +from pydantic_settings import ( + BaseSettings, + EnvSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +PLUGIN_CONFIG_FOLDER_NAME = "plugins" +# Not "plugins": on macOS the configuration and data folders are the same +# directory, and that name already holds the settings files. +PLUGIN_DATA_FOLDER_NAME = "plugin-data" +PLUGIN_ENV_PREFIX = "MVT_PLUGIN_" + +PLUGIN_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]*") + +PluginSettingsType = TypeVar("PluginSettingsType", bound="MVTPluginSettings") + + +class PluginConfigLoadError(Exception): + pass + + +def validate_plugin_name(plugin_name: Any) -> str: + """ + Check that a plugin name is safe to use in a file name and an env variable. + + :param plugin_name: Name to validate. + :returns: The validated plugin name. + """ + if not isinstance(plugin_name, str): + raise TypeError( + f"Plugin name must be a string, not {type(plugin_name).__name__}" + ) + if not PLUGIN_NAME_PATTERN.fullmatch(plugin_name): + raise ValueError( + f"Invalid plugin name {plugin_name!r}: plugin names must start with a " + "lowercase letter or a digit and may only contain lowercase letters, " + "digits and dashes" + ) + return plugin_name + + +def plugin_config_folder() -> str: + """ + Return the folder where plugins store their configuration files. + + The path is resolved on every call so it always reflects the current + environment. + """ + return os.path.join(user_config_dir("mvt"), PLUGIN_CONFIG_FOLDER_NAME) + + +def plugin_config_path(plugin_name: str) -> str: + """ + Return the path of the configuration file of a given plugin. + + :param plugin_name: Name of the plugin. + """ + return os.path.join( + plugin_config_folder(), f"{validate_plugin_name(plugin_name)}.yaml" + ) + + +def plugin_data_folder(plugin_name: str) -> str: + """ + Return the folder where a given plugin stores its data, creating it. + + Plugins should keep whatever they persist, such as caches or downloaded + artifacts, in this folder. It is created with owner-only permissions. The + path is resolved on every call so it always reflects the current + environment. A plugin with a settings class calls + `MVTPluginSettings.data_folder()` instead, which passes `plugin_name` here. + + :param plugin_name: Name of the plugin. + :returns: The path of the data folder of the plugin. + """ + # Validate the name before anything is created, so an unsafe name cannot + # leave a folder behind. + name = validate_plugin_name(plugin_name) + + # makedirs() applies its mode only to the last folder of the path, so + # MVT's own data folder keeps the default permissions while the two + # plugin folders are private. + data_folder = os.path.join(user_data_dir("mvt"), PLUGIN_DATA_FOLDER_NAME) + os.makedirs(data_folder, mode=0o700, exist_ok=True) + + folder = os.path.join(data_folder, name) + os.makedirs(folder, mode=0o700, exist_ok=True) + return folder + + +def plugin_env_prefix(plugin_name: str) -> str: + """ + Return the environment variable prefix used by a given plugin. + + Dashes are replaced by underscores. Plugin names cannot contain underscores, + so two different plugin names never share an environment namespace. + + :param plugin_name: Name of the plugin. + """ + name = validate_plugin_name(plugin_name).upper().replace("-", "_") + return f"{PLUGIN_ENV_PREFIX}{name}_" + + +def _settings_plugin_name(settings_cls: Type[BaseSettings]) -> str: + plugin_name = getattr(settings_cls, "plugin_name", None) + if plugin_name is None: + raise TypeError( + f"{settings_cls.__name__} must set a 'plugin_name' class attribute to " + "namespace its configuration file and environment variables" + ) + return validate_plugin_name(plugin_name) + + +def _plugin_yaml_source( + settings_cls: Type[BaseSettings], config_path: str +) -> YamlConfigSettingsSource: + """ + Build the YAML settings source of a plugin, reporting unusable files. + + A missing file is not an error, but a file which cannot be parsed or which + does not hold a mapping of setting names is reported with its path. + """ + try: + return YamlConfigSettingsSource(settings_cls, config_path) + except yaml.YAMLError as exc: + raise PluginConfigLoadError( + f"Invalid plugin configuration file {config_path}: {exc}" + ) from exc + except (TypeError, ValueError) as exc: + raise PluginConfigLoadError( + f"Invalid plugin configuration file {config_path}: the file must " + "contain a mapping of setting names to values" + ) from exc + + +class MVTPluginSettings(BaseSettings): + """ + Base class for plugin-namespaced settings. + + Subclass with typed fields and set `plugin_name`. Values resolve from + constructor arguments, then environment variables (MVT_PLUGIN__*), + then the plugin's YAML file (~/.config/mvt/plugins/.yaml), then field + defaults. + + Plugins must not store their settings in MVT's own configuration file: MVT + rewrites it with the fields it knows about, dropping anything else. + + `data_folder()` returns the folder the plugin keeps its data in. + """ + + model_config = SettingsConfigDict(extra="ignore") + + plugin_name: ClassVar[str] + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + plugin_name = _settings_plugin_name(cls) + # Namespace the environment variables of this plugin. Each pydantic + # model gets its own configuration dictionary, so this does not leak + # into other plugins. + cls.model_config["env_prefix"] = plugin_env_prefix(plugin_name) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + config_path = plugin_config_path(_settings_plugin_name(settings_cls)) + yaml_source = _plugin_yaml_source(settings_cls, config_path) + # Explicit arguments take precedence over environment variables, which + # in turn take precedence over the configuration file. + return (init_settings, env_settings, yaml_source) + + @classmethod + def load(cls: Type[PluginSettingsType]) -> PluginSettingsType: + """ + Load the settings of the plugin. + + A missing configuration file is not an error: the settings then come + from the environment and from the field defaults. + """ + return cls() + + @classmethod + def data_folder(cls) -> str: + """ + Return the data folder of the plugin, creating it. + + The folder is the one plugin_data_folder() returns for `plugin_name`, + so a plugin with a settings class does not repeat its name. + """ + return plugin_data_folder(_settings_plugin_name(cls)) + + def _environment_values(self) -> Dict[str, Any]: + """ + Return the settings values currently supplied by the environment. + + Values are validated by the model, so they can be compared with the + values held by this instance. + """ + settings_cls = type(self) + raw_values = EnvSettingsSource(settings_cls)() + names = [name for name in raw_values if name in settings_cls.model_fields] + if not names: + return {} + + # Fall back on the current values for the fields the environment does + # not set, so that required fields do not fail validation here. + current_values = json.loads(self.model_dump_json()) + try: + from_environment = settings_cls.model_validate( + {**current_values, **raw_values} + ) + except ValidationError: + # An environment variable which the model cannot validate must not + # stop the other environment values from being recognised, or a + # credential would be written to the configuration file. + return self._environment_values_by_field(current_values, raw_values, names) + return {name: getattr(from_environment, name) for name in names} + + def _environment_values_by_field( + self, + current_values: Dict[str, Any], + raw_values: Dict[str, Any], + names: List[str], + ) -> Dict[str, Any]: + """ + Validate each environment value on its own, skipping unusable ones. + + :param current_values: Serialized values held by this instance. + :param raw_values: Values supplied by the environment. + :param names: Names of the fields set by the environment. + """ + settings_cls = type(self) + values = {} + for name in names: + try: + from_environment = settings_cls.model_validate( + {**current_values, name: raw_values[name]} + ) + except ValidationError: + continue + values[name] = getattr(from_environment, name) + return values + + def save(self) -> None: + """ + Save the current settings to the configuration file of the plugin. + + Only values which differ from the field defaults are persisted. Values + which come from the environment are not written to disk, so credentials + passed as environment variables stay out of the configuration file. + MVT's own configuration file is never modified. + """ + config_folder = plugin_config_folder() + if not os.path.isdir(config_folder): + os.makedirs(config_folder, mode=0o700, exist_ok=True) + + values = json.loads(self.model_dump_json(exclude_defaults=True)) + for name, environment_value in self._environment_values().items(): + if name in values and getattr(self, name, None) == environment_value: + del values[name] + + # Settings files can hold credentials, so write them through a private + # temporary file and move it in place. The file is then never partially + # written and never briefly readable by other users. + config_path = plugin_config_path(self.plugin_name) + descriptor, temporary_path = tempfile.mkstemp( + dir=config_folder, prefix=f".{self.plugin_name}-", suffix=".yaml" + ) + try: + with os.fdopen(descriptor, "w") as config_file: + config_file.write(yaml.dump(values, default_flow_style=False)) + os.replace(temporary_path, config_path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(temporary_path) + raise diff --git a/src/mvt/common/updates.py b/src/mvt/common/updates.py index 001a5c2..71e97cc 100644 --- a/src/mvt/common/updates.py +++ b/src/mvt/common/updates.py @@ -3,8 +3,13 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import importlib.metadata +import json import logging import os +import re +import shlex +import subprocess from datetime import datetime from typing import Optional, Tuple @@ -12,14 +17,35 @@ import requests import yaml from packaging import version +from .cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) from .config import settings from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER +from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url from .version import MVT_VERSION log = logging.getLogger(__name__) # In hours. INDICATORS_CHECK_FREQUENCY = 12 +PLUGINS_CHECK_FREQUENCY = 12 + +# The entry-point groups a package can use to extend MVT. +PLUGIN_ENTRY_POINT_GROUPS = ( + MODULES_ENTRY_POINT_GROUP, + IOS_CLI_PLUGIN_GROUP, + ANDROID_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +SHORT_COMMIT_LENGTH = 8 +# The keys every cached finding has to carry to be printed. +FINDING_KEYS = ("name", "installed", "latest", "upgrade_command") +# Options which stop ssh from waiting for an answer nobody is there to give. +_SSH_BATCH_OPTIONS = ("-o", "BatchMode=yes", "-o", "ConnectTimeout=10") +_COMMIT_PATTERN = re.compile(r"\A[0-9a-f]{7,40}\Z") class MVTUpdates: @@ -38,6 +64,395 @@ class MVTUpdates: return "" +def installed_plugin_distributions() -> list[importlib.metadata.Distribution]: + """Return the installed distributions which extend MVT. + + A plugin package is any distribution registering at least one entry point + in the module or CLI command groups. Distributions are returned once each, + sorted by name. MVT itself is not a plugin and is never returned. + """ + distributions: dict[str, importlib.metadata.Distribution] = {} + + for group in PLUGIN_ENTRY_POINT_GROUPS: + try: + entry_points = importlib.metadata.entry_points(group=group) + except Exception as exc: + log.warning( + "Unable to discover installed plugin packages in entry-point " + "group %s: %s", + group, + exc, + ) + continue + + for entry_point in entry_points: + # Manually constructed entry points have no associated distribution. + dist = getattr(entry_point, "dist", None) + if dist is None: + continue + try: + name = dist.name + except Exception: + continue + if not name or name == "mvt": + continue + distributions.setdefault(name, dist) + + return [distributions[name] for name in sorted(distributions)] + + +def _is_usable_finding(finding: object) -> bool: + """Check that a cached finding carries everything needed to print it.""" + if not isinstance(finding, dict): + return False + + return all( + isinstance(finding.get(key), str) and finding.get(key) for key in FINDING_KEYS + ) + + +def _installed_revision( + dist: importlib.metadata.Distribution, origin: object +) -> Optional[str]: + """Return what a plugin's installed revision is right now. + + This is the value a finding recorded as installed when it was made, so a + finding can be compared against the current state of the installation + without looking anything up remotely. + """ + try: + if origin != "git": + return dist.version + + vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info") + if not isinstance(vcs_info, dict): + return None + + commit = vcs_info.get("commit_id") or "" + return commit[:SHORT_COMMIT_LENGTH] or None + except Exception as e: + log.debug("Failed to read the installed revision of a plugin: %s", e) + return None + + +def _batch_mode_ssh_command(ssh_command: str) -> str: + """Return an ssh command line which cannot stop to ask a question. + + ssh keeps the first value it is given for a keyword, so the batch mode + options are inserted right after the ssh program, ahead of whatever the + analyst configured. Their remaining options, such as which key to use, + still apply. + """ + try: + arguments = shlex.split(ssh_command.strip()) + except ValueError: + arguments = [] + + if not arguments: + arguments = ["ssh"] + + return shlex.join([arguments[0], *_SSH_BATCH_OPTIONS, *arguments[1:]]) + + +def _revision_pins_commit(revision: str, commit: str) -> bool: + """Check whether a requested revision pins the installed commit.""" + candidate = revision.lower() + if not _COMMIT_PATTERN.match(candidate): + return False + + return commit.lower().startswith(candidate) + + +class PluginUpdates: + """Check for updates to the installed MVT plugin packages. + + MVT never installs or upgrades a plugin package itself. It only reports + the command which upgrades a plugin, leaving the analyst to decide when to + run it. + """ + + @property + def latest_check_path(self) -> str: + return os.path.join(MVT_DATA_FOLDER, "latest_plugins_check") + + @property + def findings_path(self) -> str: + return os.path.join(MVT_DATA_FOLDER, "plugin_updates.json") + + def _create_data_folder(self) -> None: + if not os.path.exists(MVT_DATA_FOLDER): + os.makedirs(MVT_DATA_FOLDER) + + def get_latest_check(self) -> int: + if not os.path.exists(self.latest_check_path): + return 0 + + # A corrupt or truncated timestamp only means the next check happens + # sooner. It must never stop MVT from running. + try: + with open(self.latest_check_path, "r", encoding="utf-8") as handle: + data = handle.read().strip() + if data: + return int(data) + except (OSError, ValueError) as e: + log.debug("Failed to read the time of the latest plugin check: %s", e) + + return 0 + + def set_latest_check(self) -> None: + self._create_data_folder() + timestamp = int(datetime.now().timestamp()) + with open(self.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(timestamp)) + + def get_findings(self) -> list[dict]: + """ + Return the findings of the latest check, without checking again. + Returns an empty list if no check was ever performed. + """ + if not os.path.exists(self.findings_path): + return [] + + try: + with open(self.findings_path, "r", encoding="utf-8") as handle: + findings = json.load(handle) + except Exception as e: + log.debug("Failed to read the cached plugin updates: %s", e) + return [] + + if not isinstance(findings, list): + return [] + + # Anything which does not look like a finding is dropped rather than + # trusted: the cache is only a convenience. + return [finding for finding in findings if _is_usable_finding(finding)] + + def current_findings( + self, distributions: Optional[list[importlib.metadata.Distribution]] = None + ) -> list[dict]: + """ + Return the cached findings which still apply to what is installed. + Findings about a plugin which was upgraded or removed since the latest + check are dropped, so an update is never reported twice. + """ + if distributions is None: + distributions = installed_plugin_distributions() + + installed = {} + for dist in distributions: + try: + installed[dist.name] = dist + except Exception: + continue + + current = [] + for finding in self.get_findings(): + plugin = installed.get(finding["name"]) + if plugin is None: + continue + if ( + _installed_revision(plugin, finding.get("origin")) + != finding["installed"] + ): + continue + current.append(finding) + + return current + + def set_findings(self, findings: list[dict]) -> None: + self._create_data_folder() + with open(self.findings_path, "w", encoding="utf-8") as handle: + json.dump(findings, handle) + + def should_check(self) -> Tuple[bool, int]: + """ + Compare time of the latest plugins check with current time. + Returns bool and number of hours since the last check. + """ + now = datetime.now() + latest_check_ts = self.get_latest_check() + latest_check_dt = datetime.fromtimestamp(latest_check_ts) + + diff = now - latest_check_dt + diff_hours = divmod(diff.total_seconds(), 3600)[0] + + if diff_hours >= PLUGINS_CHECK_FREQUENCY: + return True, 0 + + return False, int(PLUGINS_CHECK_FREQUENCY - diff_hours) + + def _check_index_plugin(self, name: str, installed: str) -> Optional[dict]: + """Check a plugin installed from a package index for a newer release.""" + url = f"https://pypi.org/pypi/{name}/json" + try: + res = requests.get(url, timeout=settings.NETWORK_TIMEOUT) + except requests.exceptions.RequestException as e: + log.debug("Failed to check for updates to plugin %s: %s", name, e) + return None + + # Plugins which were never published to a public index are expected, + # and there is nothing to compare their version against. + if res.status_code == 404: + return None + + if res.status_code != 200: + log.debug( + "Failed to check for updates to plugin %s (error %d)", + name, + res.status_code, + ) + return None + + try: + latest = res.json().get("info", {}).get("version", "") + if not latest or version.parse(latest) <= version.parse(installed): + return None + except Exception as e: + log.debug("Failed to compare the versions of plugin %s: %s", name, e) + return None + + return { + "name": name, + "installed": installed, + "latest": latest, + "origin": "pypi", + # The name comes from package metadata, so the command MVT + # suggests is quoted rather than assumed to be shell-safe. + "upgrade_command": f"pip install -U {shlex.quote(name)}", + } + + def _git_ls_remote(self, url: str, revision: str) -> list[Tuple[str, str]]: + """Return the remote references matching a revision, if git allows it.""" + # Neither value is trusted: they are read from the metadata of an + # installed package and must not turn into git options. + if url.startswith("-") or revision.startswith("-"): + log.debug("Skipping the update check for the invalid repository %s", url) + return [] + + environment = dict(os.environ) + # Never prompt the analyst for repository credentials. git handles its + # own prompts, while ssh reads the terminal directly and only batch + # mode makes it fail instead of asking. + environment["GIT_TERMINAL_PROMPT"] = "0" + environment["GIT_SSH_COMMAND"] = _batch_mode_ssh_command( + environment.get("GIT_SSH_COMMAND", "") + ) + + try: + process = subprocess.run( + ["git", "ls-remote", url, revision], + capture_output=True, + stdin=subprocess.DEVNULL, + text=True, + env=environment, + timeout=settings.NETWORK_TIMEOUT, + check=False, + ) + except FileNotFoundError: + log.debug("Could not find git, skipping the update check for %s", url) + return [] + except (subprocess.SubprocessError, OSError) as e: + log.debug("Failed to query the repository %s: %s", url, e) + return [] + + if process.returncode != 0: + log.debug( + "Failed to query the repository %s (error %d): %s", + url, + process.returncode, + (process.stderr or "").strip(), + ) + return [] + + references = [] + for line in (process.stdout or "").splitlines(): + commit, _, reference = line.partition("\t") + if commit.strip() and reference.strip(): + references.append((commit.strip(), reference.strip())) + + return references + + def _check_repository_plugin( + self, name: str, direct_url: dict, vcs_info: dict + ) -> Optional[dict]: + """Check a plugin installed from a repository for a newer commit.""" + url = direct_url.get("url") or "" + installed = vcs_info.get("commit_id") or "" + revision = vcs_info.get("requested_revision") or "" + if not url or not installed: + return None + + # A plugin installed from a commit is pinned and never goes out of + # date, no matter what the branch it came from does next. + if revision and _revision_pins_commit(revision, installed): + return None + + latest = "" + wanted_reference = f"refs/heads/{revision}" if revision else "HEAD" + for commit, reference in self._git_ls_remote(url, revision or "HEAD"): + # Tags are pinned installs too. + if reference.startswith("refs/tags/"): + return None + if reference == wanted_reference: + latest = commit + + if not latest or latest == installed: + return None + + requirement = f"{name} @ git+{url}" + if revision: + requirement += f"@{revision}" + + return { + "name": name, + "installed": installed[:SHORT_COMMIT_LENGTH], + "latest": latest[:SHORT_COMMIT_LENGTH], + "origin": "git", + # A repository URL and a branch name can both hold characters a + # shell would act on, so the requirement is quoted for the shell + # the analyst is going to paste the command into. + "upgrade_command": f"pip install -U {shlex.quote(requirement)}", + } + + def _check_distribution( + self, dist: importlib.metadata.Distribution + ) -> Optional[dict]: + try: + name = dist.name + installed = dist.version + except Exception as e: + log.debug("Failed to read the metadata of an installed plugin: %s", e) + return None + + direct_url = distribution_direct_url(dist) + if direct_url is None: + return self._check_index_plugin(name, installed) + + vcs_info = direct_url.get("vcs_info") + if isinstance(vcs_info, dict): + return self._check_repository_plugin(name, direct_url, vcs_info) + + # Plugins installed from a local folder, including editable installs, + # are maintained by the analyst and have nothing to check against. + return None + + def check(self) -> list[dict]: + """ + Check every installed plugin package for an available update. + Returns one entry per plugin which can be upgraded. + """ + findings = [] + for dist in installed_plugin_distributions(): + finding = self._check_distribution(dist) + if finding: + findings.append(finding) + + self.set_findings(findings) + self.set_latest_check() + + return findings + + class IndicatorsUpdates: def __init__(self) -> None: self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}" @@ -180,9 +595,7 @@ class IndicatorsUpdates: def _get_remote_file_latest_commit( self, owner: str, repo: str, branch: str, path: str ) -> int: - file_commit_url = ( - f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" - ) + file_commit_url = f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" try: res = requests.get(file_commit_url, timeout=5) except requests.exceptions.RequestException as e: diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index 30de159..2a3e39a 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -14,7 +14,6 @@ from dataclasses import asdict, is_dataclass from typing import Any, Iterator, Union from .log import MVTLogHandler -from mvt.common.config import settings class CustomJSONEncoder(json.JSONEncoder): @@ -239,6 +238,13 @@ def init_logging(verbose: bool = False): """ log = logging.getLogger("mvt") log.setLevel(logging.DEBUG) + + # Importing an MVT CLI module calls init_logging() at import time, and + # loaded module packages may import one indirectly. Keep this idempotent + # so console log lines are not duplicated by a second handler. + if any(isinstance(handler, MVTLogHandler) for handler in log.handlers): + return + consoleHandler = MVTLogHandler() consoleHandler.setFormatter(logging.Formatter("%(message)s")) if verbose: @@ -249,16 +255,26 @@ def init_logging(verbose: bool = False): def set_verbose_logging(verbose: bool = False): + """Raise or lower the verbosity of MVT's console output. + + Only MVT's own console handler is adjusted, wherever it sits in the list. + The file handler a command attaches to its output folder keeps recording + everything, so the command.log of a run does not depend on how the run was + invoked, and a handler attached to the "mvt" logger by anything else is + left alone. + """ log = logging.getLogger("mvt") - handler = log.handlers[0] - if verbose: - handler.setLevel(logging.DEBUG) - else: - handler.setLevel(logging.INFO) + for handler in log.handlers: + if isinstance(handler, MVTLogHandler): + handler.setLevel(logging.DEBUG if verbose else logging.INFO) def exec_or_profile(module, globals, locals): """Hook for profiling MVT modules""" + # Imported here so that the CLI modules, which import this one at start-up, + # do not load the settings (and pydantic) before a command runs. + from .config import settings + if settings.PROFILE: cProfile.runctx(module, globals, locals) else: diff --git a/src/mvt/common/version.py b/src/mvt/common/version.py index c8e7716..3149604 100644 --- a/src/mvt/common/version.py +++ b/src/mvt/common/version.py @@ -3,4 +3,6 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -MVT_VERSION = "2026.7.29" +from importlib.metadata import version + +MVT_VERSION = version("mvt") diff --git a/src/mvt/ios/__init__.py b/src/mvt/ios/__init__.py index 2c05f56..6616bcb 100644 --- a/src/mvt/ios/__init__.py +++ b/src/mvt/ios/__init__.py @@ -3,4 +3,4 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -from .cli import cli +from .cli import cli, main diff --git a/src/mvt/ios/cli.py b/src/mvt/ios/cli.py index 0bd362a..56c4a11 100644 --- a/src/mvt/ios/cli.py +++ b/src/mvt/ios/cli.py @@ -14,16 +14,7 @@ from mvt.common.cli_plugins import ( load_cli_commands_option, register_cli_plugins, ) -from mvt.common.cmd_check_iocs import CmdCheckIOCS -from mvt.common.completion import ( - SUPPORTED_SHELLS, - completion_instructions, - generate_completion_script, - install_completion_script, -) -from mvt.common.logo import logo from mvt.common.options import MutuallyExclusiveOption -from mvt.common.updates import IndicatorsUpdates from mvt.common.utils import ( generate_hashes_from_path, init_logging, @@ -45,6 +36,7 @@ from mvt.common.help import ( HELP_MSG_LOAD_MODULE, HELP_MSG_MODULE, HELP_MSG_VERBOSE, + HELP_MSG_VERBOSE_COMMAND, HELP_MSG_CHECK_FS, HELP_MSG_CHECK_IOCS, HELP_MSG_STIX2, @@ -52,21 +44,17 @@ from mvt.common.help import ( HELP_MSG_CHECK_SYSDIAGNOSE, HELP_MSG_DISABLE_UPDATE_CHECK, HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, - HELP_MSG_COMPLETION, ) -from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules from mvt.common.password import prompt_password -from .cmd_check_backup import CmdIOSCheckBackup -from .cmd_check_fs import CmdIOSCheckFS -from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose -from .decrypt import ( +from .decrypt_config import ( DEFAULT_DECRYPT_WORKERS, MAX_DECRYPT_WORKERS, - DecryptBackup, ) -from .modules.backup import BACKUP_MODULES -from .modules.fs import FS_MODULES -from .modules.mixed import MIXED_MODULES + +# The commands import what they run only when they are invoked. This module is +# imported at every start of mvt-ios, including by shell completion on every +# keystroke, so importing it must do no more than build the command tree: the +# forensic modules, the backup decryption and the update checks stay out of it. init_logging() log = logging.getLogger("mvt") @@ -86,7 +74,14 @@ def _get_disable_flags(ctx): ) +def _get_verbose(ctx): + """Return whether --verbose was passed to the CLI itself.""" + return bool(ctx.obj and ctx.obj.get("verbose", False)) + + def _load_custom_modules(load_module): + from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules + try: return load_custom_modules(load_module) except CustomModuleLoadError as exc: @@ -106,60 +101,31 @@ def _load_custom_modules(load_module): is_flag=True, help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK, ) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) @click.pass_context -def cli(ctx, disable_update_check, disable_indicator_update_check): +def cli(ctx, disable_update_check, disable_indicator_update_check, verbose): ctx.ensure_object(dict) ctx.obj["disable_version_check"] = disable_update_check ctx.obj["disable_indicator_check"] = disable_indicator_update_check - if ctx.invoked_subcommand != "completion": - logo( - disable_version_check=disable_update_check, - disable_indicator_check=disable_indicator_update_check, - ) + ctx.obj["verbose"] = verbose + set_verbose_logging(verbose) + + from mvt.common.logo import logo + + logo( + disable_version_check=disable_update_check, + disable_indicator_check=disable_indicator_update_check, + ) # ============================================================================== # Command: version # ============================================================================== -@cli.command("version", help=HELP_MSG_VERSION) +@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION) def version(): return -# ============================================================================== -# Command: completion -# ============================================================================== -@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION) -@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS)) -@click.option( - "--install", - is_flag=True, - help="Write completion files and update shell configuration.", -) -@click.pass_context -def completion(ctx, shell, install): - program_name = "mvt-ios" - - if shell is None: - if install: - raise click.UsageError("A shell is required when using --install.") - click.echo(completion_instructions(program_name)) - return - - root_cli = ctx.find_root().command - - if install: - script_path = install_completion_script(root_cli, program_name, shell) - click.echo(f"Installed {shell} completion to {script_path}") - if shell in ("bash", "zsh"): - click.echo(f"Updated ~/.{shell}rc") - else: - click.echo("Fish loads completion files automatically.") - return - - click.echo(generate_completion_script(root_cli, program_name, shell)) - - # ============================================================================== # Command: decrypt-backup # ============================================================================== @@ -193,6 +159,8 @@ def completion(ctx, shell, install): @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def decrypt_backup(ctx, destination, jobs, password, key_file, hashes, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path, destination, max_workers=jobs) if key_file: @@ -256,6 +224,8 @@ def decrypt_backup(ctx, destination, jobs, password, key_file, hashes, backup_pa ) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) def extract_key(password, key_file, backup_path): + from .decrypt import DecryptBackup + backup = DecryptBackup(backup_path) if password: @@ -308,7 +278,7 @@ def extract_key(password, key_file, backup_path): help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("BACKUP_PATH", type=click.Path(exists=True)) @click.pass_context def check_backup( @@ -323,7 +293,9 @@ def check_backup( verbose, backup_path, ): - set_verbose_logging(verbose) + from .cmd_check_backup import CmdIOSCheckBackup + + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -377,7 +349,7 @@ def check_backup( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("DUMP_PATH", type=click.Path(exists=True)) @click.pass_context def check_fs( @@ -392,7 +364,9 @@ def check_fs( verbose, dump_path, ): - set_verbose_logging(verbose) + from .cmd_check_fs import CmdIOSCheckFS + + set_verbose_logging(verbose or _get_verbose(ctx)) module_options = {"fast_mode": fast} custom_modules = _load_custom_modules(load_module) @@ -446,7 +420,7 @@ def check_fs( help=HELP_MSG_LOAD_MODULE, ) @click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES) -@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE) +@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND) @click.argument("SYSDIAGNOSE_PATH", type=click.Path(exists=True)) @click.pass_context def check_sysdiagnose( @@ -460,7 +434,9 @@ def check_sysdiagnose( verbose, sysdiagnose_path, ): - set_verbose_logging(verbose) + from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose + + set_verbose_logging(verbose or _get_verbose(ctx)) custom_modules = _load_custom_modules(load_module) cmd = CmdIOSCheckSysdiagnose( target_path=sysdiagnose_path, @@ -514,6 +490,10 @@ def check_sysdiagnose( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, load_module, folder): + from mvt.common.cmd_check_iocs import CmdCheckIOCS + + from .command_modules import IOS_CHECK_IOCS_MODULES + custom_modules = _load_custom_modules(load_module) cmd = CmdCheckIOCS( target_path=folder, @@ -524,7 +504,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): custom_modules=custom_modules, platform="ios", ) - cmd.modules = BACKUP_MODULES + FS_MODULES + MIXED_MODULES + cmd.modules = IOS_CHECK_IOCS_MODULES if list_modules: cmd.list_modules() @@ -540,12 +520,25 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder): # ============================================================================== @cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2) def download_iocs(): + from mvt.common.updates import IndicatorsUpdates + ioc_updates = IndicatorsUpdates() ioc_updates.update() -register_cli_plugins( - cli, - entry_point_group=IOS_CLI_PLUGIN_GROUP, - environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV, -) +# ============================================================================== +# Entry point of the mvt-ios console script +# ============================================================================== +def main() -> None: + """Register the external commands and run the mvt-ios CLI. + + External commands are registered here rather than when this module is + imported, so that importing MVT never runs third-party code and a plugin + importing from MVT cannot re-enter a module that is still initializing. + """ + register_cli_plugins( + cli, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV, + ) + cli() diff --git a/src/mvt/ios/command_modules.py b/src/mvt/ios/command_modules.py new file mode 100644 index 0000000..26fa1d6 --- /dev/null +++ b/src/mvt/ios/command_modules.py @@ -0,0 +1,22 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Module lists an mvt-ios command composes from more than one family. + +Commands whose modules are one family read that family directly. check-iocs +re-checks stored results, so it has to know every module that could have +written one, and both the CLI and any other code needing that answer share +the list from here rather than each concatenating their own. +""" + +from mvt.common.module import MVTModule + +from .modules.backup import BACKUP_MODULES +from .modules.fs import FS_MODULES +from .modules.mixed import MIXED_MODULES + +IOS_CHECK_IOCS_MODULES: list[type[MVTModule]] = ( + BACKUP_MODULES + FS_MODULES + MIXED_MODULES +) diff --git a/src/mvt/ios/data/ios_versions.json b/src/mvt/ios/data/ios_versions.json index 16fbba5..9b4613b 100644 --- a/src/mvt/ios/data/ios_versions.json +++ b/src/mvt/ios/data/ios_versions.json @@ -1220,6 +1220,10 @@ "version": "18.7.10", "build": "22H373" }, + { + "version": "18.7.10", + "build": "22H374" + }, { "version": "26", "build": "23A341" @@ -1263,5 +1267,9 @@ { "version": "26.6.1", "build": "23G82" + }, + { + "version": "26.6.1", + "build": "23G83" } ] \ No newline at end of file diff --git a/src/mvt/ios/decrypt.py b/src/mvt/ios/decrypt.py index faf3000..dc30d9c 100644 --- a/src/mvt/ios/decrypt.py +++ b/src/mvt/ios/decrypt.py @@ -26,10 +26,9 @@ from iphone_backup_decrypt import EncryptedBackup from iphone_backup_decrypt import google_iphone_dataprotection from iphone_backup_decrypt.utils import FilePlist -log = logging.getLogger(__name__) +from .decrypt_config import DEFAULT_DECRYPT_WORKERS, MAX_DECRYPT_WORKERS -DEFAULT_DECRYPT_WORKERS = 4 -MAX_DECRYPT_WORKERS = 32 +log = logging.getLogger(__name__) # Import pbkdf2_hmac from the same source iphone_backup_decrypt uses internally, # so our key derivation is consistent with theirs. diff --git a/src/mvt/ios/decrypt_config.py b/src/mvt/ios/decrypt_config.py new file mode 100644 index 0000000..ef153e9 --- /dev/null +++ b/src/mvt/ios/decrypt_config.py @@ -0,0 +1,9 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Configuration shared by the iOS CLI and backup decryption implementation.""" + +DEFAULT_DECRYPT_WORKERS = 4 +MAX_DECRYPT_WORKERS = 32 diff --git a/src/mvt/ios/modules/backup/__init__.py b/src/mvt/ios/modules/backup/__init__.py index 0fb833a..ce8499b 100644 --- a/src/mvt/ios/modules/backup/__init__.py +++ b/src/mvt/ios/modules/backup/__init__.py @@ -3,9 +3,16 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .backup_info import BackupInfo from .configuration_profiles import ConfigurationProfiles from .manifest import Manifest from .profile_events import ProfileEvents -BACKUP_MODULES = [BackupInfo, ConfigurationProfiles, Manifest, ProfileEvents] +BACKUP_MODULES: list[type[MVTModule]] = [ + BackupInfo, + ConfigurationProfiles, + Manifest, + ProfileEvents, +] diff --git a/src/mvt/ios/modules/fs/__init__.py b/src/mvt/ios/modules/fs/__init__.py index 06dba24..b00ee35 100644 --- a/src/mvt/ios/modules/fs/__init__.py +++ b/src/mvt/ios/modules/fs/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .analytics import Analytics from .analytics_ios_versions import AnalyticsIOSVersions from .cache_files import CacheFiles @@ -15,7 +17,7 @@ from .webkit_indexeddb import WebkitIndexedDB from .webkit_localstorage import WebkitLocalStorage from .webkit_safariviewservice import WebkitSafariViewService -FS_MODULES = [ +FS_MODULES: list[type[MVTModule]] = [ CacheFiles, Filesystem, Netusage, diff --git a/src/mvt/ios/modules/mixed/__init__.py b/src/mvt/ios/modules/mixed/__init__.py index 0e50c26..027dce0 100644 --- a/src/mvt/ios/modules/mixed/__init__.py +++ b/src/mvt/ios/modules/mixed/__init__.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +from mvt.common.module import MVTModule + from .applications import Applications from .calendar import Calendar from .calls import Calls @@ -26,8 +28,9 @@ from .tcc import TCC from .webkit_resource_load_statistics import WebkitResourceLoadStatistics from .webkit_session_resource_log import WebkitSessionResourceLog from .whatsapp import Whatsapp +from .whatsapp_contacts import WhatsappContacts -MIXED_MODULES = [ +MIXED_MODULES: list[type[MVTModule]] = [ Calls, ChromeFavicon, ChromeHistory, @@ -47,6 +50,7 @@ MIXED_MODULES = [ WebkitResourceLoadStatistics, WebkitSessionResourceLog, Whatsapp, + WhatsappContacts, Shortcuts, Applications, Calendar, diff --git a/src/mvt/ios/modules/mixed/interactionc.py b/src/mvt/ios/modules/mixed/interactionc.py index 81a67e2..4d21382 100644 --- a/src/mvt/ios/modules/mixed/interactionc.py +++ b/src/mvt/ios/modules/mixed/interactionc.py @@ -3,9 +3,11 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import datetime import logging +import re import sqlite3 -from typing import Optional +from typing import Optional, Tuple from mvt.common.module_types import ( ModuleAtomicResult, @@ -15,6 +17,7 @@ from mvt.common.module_types import ( from mvt.common.utils import convert_mactime_to_iso from ..base import IOSExtraction +from .whatsapp_contacts import WhatsappContacts INTERACTIONC_BACKUP_IDS = [ "1f5a521220a3ad80ebfdc196978df8e7a2e49dee", @@ -22,6 +25,49 @@ INTERACTIONC_BACKUP_IDS = [ INTERACTIONC_ROOT_PATHS = [ "private/var/mobile/Library/CoreDuet/People/interactionC.db", ] + +# The interaction record's creation date normally trails its start date by +# milliseconds: emitting it as a timeline event only duplicates the start +# date event. A large divergence, however, indicates the record was +# backfilled (sync, restore, or tampering) and is worth surfacing. +CREATION_DATE_DIVERGENCE_THRESHOLD = 3600.0 + +# Per-contact aggregate dates from ZCONTACTS are repeated on every +# interaction row of the same contact. They are serialized with a +# contact-centric data string so that timeline de-duplication collapses +# them into one event per contact. +CONTACT_EVENT_TEMPLATES = { + "contacts_creation_date": "Contact {party} first recorded in interactionC", + "first_incoming_sender_date": "First incoming interaction from {party}", + "last_incoming_sender_date": "Last incoming interaction from {party}", + "first_incoming_recipient_date": ( + "First incoming interaction where {party} was a recipient" + ), + "last_incoming_recipient_date": ( + "Last incoming interaction where {party} was a recipient" + ), + "first_outgoing_recipient_date": ( + "First outgoing interaction to {party}" + ), + "last_outgoing_recipient_date": ( + "Last outgoing interaction to {party}" + ), +} + + +def _parse_iso(timestamp) -> Optional[datetime.datetime]: + try: + return datetime.datetime.strptime( + timestamp, "%Y-%m-%d %H:%M:%S.%f" + ) + except (TypeError, ValueError): + return None + + +def _describe_delta(seconds: float) -> str: + if seconds >= 86400: + return f"{seconds / 86400:.0f} days" + return f"{seconds / 3600:.0f} hours" # Taken from APOLLO # https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt QUERIES = [ @@ -34,7 +80,7 @@ QUERIES = [ CASE ZINTERACTIONS.ZDIRECTION WHEN '0' THEN 'INCOMING' WHEN '1' THEN 'OUTGOING' - END 'DIRECTION' AS "direction", + END AS "direction", ZCONTACTS.ZDISPLAYNAME AS "sender_display_name", ZCONTACTS.ZIDENTIFIER AS "sender_identifier", ZCONTACTS.ZPERSONID AS "sender_personid", @@ -89,7 +135,7 @@ QUERIES = [ CASE ZINTERACTIONS.ZDIRECTION WHEN '0' THEN 'INCOMING' WHEN '1' THEN 'OUTGOING' - END 'DIRECTION' AS "direction", + END AS "direction", ZCONTACTS.ZDISPLAYNAME AS "sender_display_name", ZCONTACTS.ZIDENTIFIER AS "sender_identifier", ZCONTACTS.ZPERSONID AS "sender_personid", @@ -117,7 +163,7 @@ QUERIES = [ CASE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE WHEN '0' THEN '0' ELSE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE - END 'LAST INCOMING RECIPIENT DATE' AS "last_incoming_recipient_date", + END AS "last_incoming_recipient_date", ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE AS "last_outgoing_recipient_date", ZCONTACTS.ZCUSTOMIDENTIFIER AS "custom_id", ZINTERACTIONS.ZCONTENTURL AS "interaction_content_url", @@ -218,9 +264,16 @@ QUERIES = [ ] +WHATSAPP_BUNDLE_ID = "net.whatsapp.WhatsApp" + + class InteractionC(IOSExtraction): """This module extracts data from InteractionC db.""" + # WhatsApp identifies chat peers by LID in interactionC.db, which only the + # WhatsApp contacts database can map back to a phone number and name. + dependencies = [WhatsappContacts] + def __init__( self, file_path: Optional[str] = None, @@ -252,10 +305,51 @@ class InteractionC(IOSExtraction): "last_outgoing_recipient_date", ] + @staticmethod + def _describe_party(record: ModuleAtomicResult, prefix: str) -> Optional[str]: + name = record.get(f"{prefix}_display_name") or record.get( + f"{prefix}_resolved_name" + ) + identifier = record.get(f"{prefix}_resolved_phone_number") or record.get( + f"{prefix}_identifier" + ) + if name and identifier: + # A display name that is just a formatted copy of the phone + # number adds no information. + name_digits = re.sub(r"\D", "", name) + if name_digits and name_digits == re.sub(r"\D", "", identifier): + return identifier + return f"{name} ({identifier})" + return name or identifier or None + def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + sender = self._describe_party(record, "sender") + # The chat peer from the domain identifier stands in when the + # recipient was not recorded (or the recipient join is unavailable). + recipient = self._describe_party(record, "recipient") or self._describe_party( + record, "domain" + ) + direction = record.get("direction") + if not sender and direction == "OUTGOING": + sender = "local user" + if not recipient and direction == "INCOMING": + recipient = "local user" + + header = f"[{record['bundle_id']}]" + if record.get("account"): + header += f" {record['account']}" + if direction: + header += f" {direction}" + + data = f"{header} from {sender or 'unknown'} to {recipient or 'unknown'}" + if record.get("group_name"): + data += f" (group: {record['group_name']})" + if record.get("content"): + data += f": {record['content']}" + records = [] processed = [] - for timestamp in self.timestamps: + for timestamp in ("start_date", "end_date"): # Check if the record has the current timestamp. if timestamp not in record or not record[timestamp]: continue @@ -269,16 +363,142 @@ class InteractionC(IOSExtraction): "timestamp": record[timestamp], "module": self.__class__.__name__, "event": timestamp, - "data": f"[{record['bundle_id']}] {record['account']} - " - f"from {record['sender_display_name']} ({record['sender_identifier']}) " - f"to {record.get('recipient_display_name', '')} ({record.get('recipient_identifier', '')}):" - f" {record.get('content', '')}", + "data": data, } ) processed.append(record[timestamp]) + creation_event = self._serialize_creation_date(record, data) + if creation_event: + records.append(creation_event) + + # Contact-level aggregates describe the sender's contact record. + party = self._describe_party(record, "sender") + if party: + for field, template in CONTACT_EVENT_TEMPLATES.items(): + if not record.get(field): + continue + records.append( + { + "timestamp": record[field], + "module": self.__class__.__name__, + "event": field, + "data": template.format(party=party), + } + ) + return records + def _serialize_creation_date( + self, record: ModuleAtomicResult, data: str + ) -> Optional[dict]: + """Serialize the interaction record's creation date only when it + diverges from the start date enough to indicate the record was + backfilled.""" + creation = record.get("interactions_creation_date") + if not creation: + return None + + event = { + "timestamp": creation, + "module": self.__class__.__name__, + "event": "interactions_creation_date", + "data": data, + } + + start = _parse_iso(record.get("start_date")) + creation_parsed = _parse_iso(creation) + if not start or not creation_parsed: + # Without a start date the creation date is the only anchor. + return event + + delta = (creation_parsed - start).total_seconds() + if abs(delta) < CREATION_DATE_DIVERGENCE_THRESHOLD: + return None + + direction = "after" if delta > 0 else "before" + event["data"] = ( + f"Interaction record created {_describe_delta(abs(delta))} " + f"{direction} the event: {data}" + ) + return event + + def _whatsapp_contact_maps(self) -> Tuple[dict, dict]: + """Build LID and phone-digit lookup maps from the WhatsappContacts + module results, when available.""" + by_lid: dict = {} + by_phone: dict = {} + contacts_module = self.dependency_modules.get(WhatsappContacts) + if not contacts_module: + return by_lid, by_phone + + for contact in contacts_module.results: + name = contact.get("full_name") or contact.get("given_name") + phone = contact.get("phone_number") + entry = (phone, name) + if contact.get("lid"): + by_lid[contact["lid"]] = entry + if phone: + by_phone[re.sub(r"\D", "", phone)] = entry + whatsapp_id = contact.get("whatsapp_id") + if whatsapp_id and "@" in whatsapp_id: + by_phone.setdefault(whatsapp_id.split("@")[0], entry) + + return by_lid, by_phone + + @staticmethod + def _resolve_whatsapp_identifier( + value, by_lid: dict, by_phone: dict + ) -> Tuple[Optional[str], Optional[str]]: + """Resolve a WhatsApp identifier (LID, JID or phone number) to a + (phone_number, contact_name) tuple.""" + if not value: + return None, None + + value = str(value) + if value.endswith("@lid"): + return by_lid.get(value, (None, None)) + if value.endswith("@g.us"): + return None, None + if value.endswith("@s.whatsapp.net"): + digits = value.split("@")[0] + phone, name = by_phone.get(digits, (None, None)) + return phone or f"+{digits}", name + if value.startswith("+"): + _, name = by_phone.get(re.sub(r"\D", "", value), (None, None)) + return None, name + + return None, None + + def _postprocess_results(self) -> None: + by_lid, by_phone = self._whatsapp_contact_maps() + + for entry in self.results: + # The fallback queries return ZDIRECTION raw instead of labelled. + if entry.get("direction") in (0, "0"): + entry["direction"] = "INCOMING" + elif entry.get("direction") in (1, "1"): + entry["direction"] = "OUTGOING" + + if entry.get("bundle_id") != WHATSAPP_BUNDLE_ID: + continue + + candidates = { + "sender": [entry.get("sender_identifier"), entry.get("custom_id")], + "recipient": [entry.get("recipient_identifier")], + "domain": [entry.get("domain_identifier")], + } + for prefix, values in candidates.items(): + phone = name = None + for value in values: + phone, name = self._resolve_whatsapp_identifier( + value, by_lid, by_phone + ) + if phone or name: + break + entry[f"{prefix}_resolved_phone_number"] = phone + entry[f"{prefix}_resolved_name"] = name + def run(self) -> None: self._find_ios_database( backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS @@ -325,4 +545,6 @@ class InteractionC(IOSExtraction): cur.close() conn.close() + self._postprocess_results() + self.log.info("Extracted a total of %d InteractionC events", len(self.results)) diff --git a/src/mvt/ios/modules/mixed/whatsapp.py b/src/mvt/ios/modules/mixed/whatsapp.py index 0a80aad..89601df 100644 --- a/src/mvt/ios/modules/mixed/whatsapp.py +++ b/src/mvt/ios/modules/mixed/whatsapp.py @@ -4,7 +4,9 @@ # https://license.mvt.re/1.1/ import logging -from typing import Optional +import os +import sqlite3 +from typing import Dict, Optional from mvt.common.module_types import ( ModuleAtomicResult, @@ -22,9 +24,71 @@ WHATSAPP_ROOT_PATHS = [ "private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite", ] +WHATSAPP_LID_BACKUP_IDS = [ + # SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-LID.sqlite" + "e794f6ffcc3c222535f47684a63d5178da3c4500", +] +WHATSAPP_LID_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/LID.sqlite", +] + +# WhatsApp records the mapping between a contact's LID and phone number +# identifiers in the ZWAPHONENUMBERLIDPAIR table. Depending on the WhatsApp +# version this lives in a dedicated LID.sqlite database or in +# ChatStorage.sqlite itself. +LID_PAIRS_QUERY = """ + SELECT + ZLID AS "lid", + ZPHONENUMBER AS "phone_number", + ZTIMESTAMP AS "pair_timestamp" + FROM ZWAPHONENUMBERLIDPAIR; +""" + +CHAT_SESSIONS_QUERY = """ + SELECT + ZWACHATSESSION.Z_PK AS "session_pk", + ZWACHATSESSION.ZCONTACTJID AS "contact_jid", + ZWACHATSESSION.ZPARTNERNAME AS "partner_name", + ZWACHATSESSION.ZSESSIONTYPE AS "session_type", + ZWACHATSESSION.ZARCHIVED AS "archived", + ZWACHATSESSION.ZREMOVED AS "removed", + ZWACHATSESSION.ZMESSAGECOUNTER AS "message_counter", + ZWACHATSESSION.ZLASTMESSAGEDATE AS "last_message_date", + ZWAGROUPINFO.ZCREATIONDATE AS "group_creation_date", + MIN(ZWAMESSAGE.ZMESSAGEDATE) AS "first_stored_message_date", + MAX(ZWAMESSAGE.ZMESSAGEDATE) AS "last_stored_message_date", + COUNT(ZWAMESSAGE.Z_PK) AS "stored_message_count" + FROM ZWACHATSESSION + LEFT JOIN ZWAGROUPINFO + ON ZWACHATSESSION.ZGROUPINFO = ZWAGROUPINFO.Z_PK + LEFT JOIN ZWAMESSAGE + ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK + GROUP BY ZWACHATSESSION.Z_PK; +""" + +CHAT_SESSION_DATE_FIELDS = [ + "last_message_date", + "group_creation_date", + "first_stored_message_date", + "last_stored_message_date", +] + + +def _describe_chat(record: ModuleAtomicResult) -> str: + jid = record.get("contact_jid") or "unknown" + name = record.get("partner_name") + identifier = record.get("partner_resolved_phone_number") or jid + label = f"'{name}' ({identifier})" if name else identifier + is_group = jid.endswith("@g.us") or record.get("group_creation_date") + if is_group: + return f"WhatsApp group chat {label}" + return f"WhatsApp chat with {label}" + class Whatsapp(IOSExtraction): - """This module extracts all WhatsApp messages containing links.""" + """This module extracts all WhatsApp messages containing links, as well + as per-chat records with the first and last interaction dates of each + conversation.""" def __init__( self, @@ -45,6 +109,21 @@ class Whatsapp(IOSExtraction): ) def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + if record.get("record_type") == "chat_session": + return self._serialize_chat_session(record) + if record.get("record_type") == "lid_phone_number_pair": + if not record.get("pair_timestamp"): + return [] + return { + "timestamp": record["pair_timestamp"], + "module": self.__class__.__name__, + "event": "lid_pair_recorded", + "data": ( + f"WhatsApp associated LID {record.get('lid')} with " + f"phone number {record.get('phone_number')}" + ), + } + text = record.get("ZTEXT", "").replace("\n", "\\n") links_text = "" if record.get("links"): @@ -57,6 +136,49 @@ class Whatsapp(IOSExtraction): "data": f"'{text}' from {record.get('ZFROMJID', 'Unknown')}{links_text}", } + def _serialize_chat_session( + self, record: ModuleAtomicResult + ) -> ModuleSerializedResult: + records = [] + chat = _describe_chat(record) + + if record.get("group_creation_date"): + records.append( + { + "timestamp": record["group_creation_date"], + "module": self.__class__.__name__, + "event": "group_created", + "data": f"{chat} was created", + } + ) + + if record.get("first_stored_message_date"): + records.append( + { + "timestamp": record["first_stored_message_date"], + "module": self.__class__.__name__, + "event": "chat_first_message", + "data": f"First stored message in {chat}", + } + ) + + # The chat session's own last-message date is authoritative: it can + # postdate the newest stored message if that message was deleted. + last_message_date = record.get("last_message_date") or record.get( + "last_stored_message_date" + ) + if last_message_date: + records.append( + { + "timestamp": last_message_date, + "module": self.__class__.__name__, + "event": "chat_last_message", + "data": f"Last message in {chat}", + } + ) + + return records + def check_indicators(self) -> None: if not self.indicators: return @@ -146,7 +268,126 @@ class Whatsapp(IOSExtraction): message["links"] = list(set(filtered_links)) self.results.append(message) + total_messages = len(self.results) + lid_map = self._extract_lid_pairs(cur) + total_sessions = self._extract_chat_sessions(cur, lid_map) + cur.close() conn.close() - self.log.info("Extracted a total of %d WhatsApp messages", len(self.results)) + self.log.info( + "Extracted a total of %d WhatsApp messages, %d chat sessions " + "and %d LID-phone number pairs", + total_messages, + total_sessions, + len(lid_map), + ) + + def _find_lid_db_path(self) -> Optional[str]: + for backup_id in WHATSAPP_LID_BACKUP_IDS: + file_path = self._get_backup_file_from_id(backup_id) + if file_path and os.path.exists(file_path): + return file_path + for found_path in self._get_fs_files_from_patterns( + WHATSAPP_LID_ROOT_PATHS + ): + return found_path + return None + + def _extract_lid_pairs(self, chat_cur: sqlite3.Cursor) -> Dict[str, str]: + """Extract the LID to phone number mapping from the dedicated + LID.sqlite database, falling back to the same table in + ChatStorage.sqlite. Returns a map of LID digits to phone number + digits.""" + rows = [] + lid_db_path = self._find_lid_db_path() + if lid_db_path: + self.log.info( + "Found WhatsApp LID database at path: %s", lid_db_path + ) + lid_conn = self._open_sqlite_db(lid_db_path) + try: + lid_cur = lid_conn.cursor() + lid_cur.execute(LID_PAIRS_QUERY) + rows = lid_cur.fetchall() + lid_cur.close() + except sqlite3.DatabaseError as exc: + self.log.warning( + "Unable to extract WhatsApp LID-phone number pairs: %s", + exc, + ) + finally: + lid_conn.close() + else: + try: + chat_cur.execute(LID_PAIRS_QUERY) + rows = chat_cur.fetchall() + except sqlite3.OperationalError: + self.log.info( + "No WhatsApp LID database found in this backup or " + "filesystem dump: LID chat identifiers cannot be " + "resolved to phone numbers" + ) + + lid_map: Dict[str, str] = {} + for lid, phone_number, pair_timestamp in rows: + record = { + "record_type": "lid_phone_number_pair", + "lid": lid, + "phone_number": phone_number, + "pair_timestamp": ( + convert_mactime_to_iso(pair_timestamp) or None + ) + if pair_timestamp + else None, + } + self.results.append(record) + + if lid and phone_number: + lid_digits = str(lid).split("@")[0] + phone_digits = str(phone_number).split("@")[0].lstrip("+") + lid_map[lid_digits] = phone_digits + + return lid_map + + def _extract_chat_sessions( + self, cur: sqlite3.Cursor, lid_map: Dict[str, str] + ) -> int: + """Extract one record per chat session with the first and last + interaction dates of each conversation.""" + try: + cur.execute(CHAT_SESSIONS_QUERY) + except sqlite3.OperationalError as exc: + self.log.warning( + "Unable to extract WhatsApp chat sessions: %s", exc + ) + return 0 + + names = [description[0] for description in cur.description] + + total_sessions = 0 + for row in cur.fetchall(): + session = dict(zip(names, row)) + session["record_type"] = "chat_session" + + session["partner_resolved_phone_number"] = None + jid = session.get("contact_jid") or "" + if jid.endswith("@lid"): + phone_digits = lid_map.get(jid.split("@")[0]) + if phone_digits: + session["partner_resolved_phone_number"] = ( + f"+{phone_digits}" + ) + + for field in CHAT_SESSION_DATE_FIELDS: + if session.get(field): + session[field] = ( + convert_mactime_to_iso(session[field]) or None + ) + else: + session[field] = None + + self.results.append(session) + total_sessions += 1 + + return total_sessions diff --git a/src/mvt/ios/modules/mixed/whatsapp_contacts.py b/src/mvt/ios/modules/mixed/whatsapp_contacts.py new file mode 100644 index 0000000..309da22 --- /dev/null +++ b/src/mvt/ios/modules/mixed/whatsapp_contacts.py @@ -0,0 +1,308 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +import sqlite3 +from typing import Optional + +from mvt.common.module import DatabaseNotFoundError +from mvt.common.module_types import ( + ModuleAtomicResult, + ModuleResults, + ModuleSerializedResult, +) +from mvt.common.utils import convert_mactime_to_iso + +from ..base import IOSExtraction + +WHATSAPP_CONTACTS_BACKUP_IDS = [ + # SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-ContactsV2.sqlite" + "b8548dc30aa1030df0ce18ef08b882cf7ab5212f", +] +WHATSAPP_CONTACTS_ROOT_PATHS = [ + "private/var/mobile/Containers/Shared/AppGroup/*/ContactsV2.sqlite", +] + +# WhatsApp's standard disappearing-messages timer values, in seconds. +DISAPPEARING_DURATION_LABELS = { + 86400: "24 hours", + 604800: "7 days", + 1209600: "14 days", + 2592000: "30 days", + 7776000: "90 days", +} + +# Output field -> candidate columns in ZWAADDRESSBOOKCONTACT, in order of +# preference. WhatsApp renames columns across versions, so the query is built +# from the columns actually present in the database. +COLUMN_CANDIDATES = { + "whatsapp_id": ["ZWHATSAPPID"], + "lid": ["ZLID"], + "phone_number": ["ZPHONENUMBER"], + "localized_phone_number": ["ZLOCALIZEDPHONENUMBER"], + "full_name": ["ZFULLNAME"], + "given_name": ["ZGIVENNAME"], + "last_name": ["ZLASTNAME"], + "user_name": ["ZUSERNAME"], + "business_name": ["ZBUSINESSNAME"], + "about_text": ["ZABOUTTEXT"], + "about_emoji": ["ZABOUTEMOJI"], + "notes": ["ZNOTES"], + "disappearing_mode_duration": ["ZDISAPPEARINGMODEDURATION"], + "disappearing_mode_timestamp": ["ZDISAPPEARINGMODETIMESTAMP"], + "about_timestamp": ["ZABOUTTIMESTAMP"], + "about_expiration_timestamp": ["ZABOUTEXPIRATIONTIMESTAMP"], + "last_updated": ["ZLASTUPDATED"], + "phone_status": ["ZPHONESTATUS", "ZPHONENUMBERSTATUS"], + "sync_policy": ["ZSYNCPOLICY"], +} + +STRING_FIELDS = [ + "whatsapp_id", + "lid", + "phone_number", + "localized_phone_number", + "full_name", + "given_name", + "last_name", + "user_name", + "business_name", + "about_text", + "about_emoji", + "notes", +] + +DATE_FIELDS = [ + "disappearing_mode_timestamp", + "about_timestamp", + "about_expiration_timestamp", + "last_updated", +] + + +def _decode_string(value) -> Optional[str]: + # CoreData stores string attributes as UTF-8 blobs in some WhatsApp + # versions, so values can arrive as either bytes or str. + if value is None: + return None + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) + + +def _label_duration(duration) -> str: + if not duration: + return "off" + return DISAPPEARING_DURATION_LABELS.get( + int(duration), f"{int(duration)} seconds" + ) + + +def _describe_contact(record: ModuleAtomicResult) -> str: + contact = ( + record.get("whatsapp_id") + or record.get("lid") + or record.get("phone_number") + or "unknown" + ) + full_name = record.get("full_name") + if full_name: + contact = f"{contact} ({full_name})" + return contact + + +class WhatsappContacts(IOSExtraction): + """This module extracts WhatsApp contact records and per-contact + disappearing-messages settings from ContactsV2.sqlite. + + ChatStorage.sqlite does not record the disappearing-messages state of 1:1 + chats: the authoritative timer is stored on each contact record in this + database, alongside the mapping between a contact's LID and phone number + identifiers. + """ + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[ModuleResults] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: + records = [] + contact = _describe_contact(record) + + if record.get("disappearing_mode_timestamp"): + records.append( + { + "timestamp": record["disappearing_mode_timestamp"], + "module": self.__class__.__name__, + "event": "disappearing_mode_set", + "data": ( + f"WhatsApp disappearing messages timer set to " + f"'{record.get('disappearing_mode_label')}' " + f"for {contact}" + ), + } + ) + + if record.get("about_timestamp"): + data = f"WhatsApp about text of {contact} changed" + about_text = record.get("about_text") + if about_text: + data += f' to "{about_text}"' + records.append( + { + "timestamp": record["about_timestamp"], + "module": self.__class__.__name__, + "event": "about_changed", + "data": data, + } + ) + + if record.get("about_expiration_timestamp"): + records.append( + { + "timestamp": record["about_expiration_timestamp"], + "module": self.__class__.__name__, + "event": "about_expiration", + "data": ( + f"WhatsApp about text of {contact} scheduled " + f"to expire" + ), + } + ) + + if record.get("last_updated"): + records.append( + { + "timestamp": record["last_updated"], + "module": self.__class__.__name__, + "event": "contact_last_updated", + "data": f"WhatsApp contact record for {contact} updated", + } + ) + + return records + + def run(self) -> None: + try: + self._find_ios_database( + backup_ids=WHATSAPP_CONTACTS_BACKUP_IDS, + root_paths=WHATSAPP_CONTACTS_ROOT_PATHS, + ) + except DatabaseNotFoundError: + self.log.warning( + "Unable to find the WhatsApp ContactsV2.sqlite database in " + "this backup or filesystem dump. WhatsApp disappearing " + "messages settings and contact records cannot be extracted. " + "This database is often missing from incremental backups." + ) + return + + self.log.info( + "Found WhatsApp contacts database at path: %s", self.file_path + ) + + assert self.file_path is not None + conn = self._open_sqlite_db(self.file_path) + cur = conn.cursor() + try: + try: + cur.execute("PRAGMA table_info(ZWAADDRESSBOOKCONTACT)") + available_columns = {row[1] for row in cur.fetchall()} + except sqlite3.DatabaseError as exc: + self.log.error( + "Unable to read the ZWAADDRESSBOOKCONTACT table schema: %s", + exc, + ) + return + + if not available_columns: + self.log.warning( + "The WhatsApp contacts database does not contain a " + "ZWAADDRESSBOOKCONTACT table" + ) + return + + selected = {} + for field, candidates in COLUMN_CANDIDATES.items(): + for candidate in candidates: + if candidate in available_columns: + selected[field] = candidate + break + + # A record with no duration column is "unknown", not "off": the + # timer state cannot be determined from this database version. + has_duration = "disappearing_mode_duration" in selected + if not has_duration: + self.log.warning( + "The ZDISAPPEARINGMODEDURATION column is not present in " + "this WhatsApp contacts database: disappearing messages " + "state is unknown" + ) + + columns = ["Z_PK"] + list(selected.values()) + cur.execute( + f"SELECT {', '.join(columns)} FROM ZWAADDRESSBOOKCONTACT;" + ) + fields = ["row_pk"] + list(selected.keys()) + + for row in cur: + record = dict(zip(fields, row)) + + for field in STRING_FIELDS: + if field in record: + record[field] = _decode_string(record[field]) + else: + record[field] = None + + for field in DATE_FIELDS: + if record.get(field) is not None: + record[field] = ( + convert_mactime_to_iso(record[field]) or None + ) + else: + record[field] = None + + duration = record.get("disappearing_mode_duration") + if has_duration: + record["disappearing_mode_is_on"] = bool(duration) + record["disappearing_mode_label"] = _label_duration( + duration + ) + else: + record["disappearing_mode_duration"] = None + record["disappearing_mode_is_on"] = None + record["disappearing_mode_label"] = None + + record.setdefault("phone_status", None) + record.setdefault("sync_policy", None) + + self.results.append(record) + finally: + cur.close() + conn.close() + + total_ephemeral = sum( + 1 for record in self.results if record["disappearing_mode_is_on"] + ) + self.log.info( + "Extracted a total of %d WhatsApp contacts (%d with disappearing " + "messages enabled)", + len(self.results), + total_ephemeral, + ) diff --git a/src/mvt/plugin.py b/src/mvt/plugin.py new file mode 100644 index 0000000..9257dc6 --- /dev/null +++ b/src/mvt/plugin.py @@ -0,0 +1,80 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Stable functions and modules which plugins can import. + +Anything else in mvt can be imported too, and may change +between releases without notice. +""" + +from mvt.android.modules.androidqf.base import AndroidQFModule +from mvt.android.modules.backup.base import BackupModule as AndroidBackupModule +from mvt.android.modules.bugreport.base import BugReportModule +from mvt.common.alerts import Alert, AlertLevel +from mvt.common.command import Command +from mvt.common.config import settings +from mvt.common.module import DatabaseCorruptedError, DatabaseNotFoundError, MVTModule +from mvt.common.module_loader import get_plugin_logger +from mvt.common.module_types import ( + ModuleAtomicResult, + ModuleResults, + ModuleSerializedResult, +) +from mvt.common.plugin_config import ( + MVTPluginSettings, + PluginConfigLoadError, + plugin_config_path, + plugin_data_folder, + plugin_env_prefix, +) +from mvt.common.utils import ( + convert_chrometime_to_datetime, + convert_datetime_to_iso, + convert_mactime_to_datetime, + convert_mactime_to_iso, + convert_unix_to_iso, + convert_unix_to_utc_datetime, +) +from mvt.common.version import MVT_VERSION +from mvt.ios.modules.base import IOSExtraction +from mvt.ios.modules.sysdiagnose.base import SysdiagnoseExtraction + +__all__ = [ + # Classes a plugin subclasses. + "MVTModule", + "IOSExtraction", + "SysdiagnoseExtraction", + "AndroidQFModule", + "AndroidBackupModule", + "BugReportModule", + "Command", + # Results and alerts. + "ModuleAtomicResult", + "ModuleResults", + "ModuleSerializedResult", + "Alert", + "AlertLevel", + # Errors a module raises. + "DatabaseNotFoundError", + "DatabaseCorruptedError", + # Settings. + "settings", + "MVTPluginSettings", + "PluginConfigLoadError", + "plugin_config_path", + "plugin_data_folder", + "plugin_env_prefix", + # Logging. + "get_plugin_logger", + # Timestamps. + "convert_chrometime_to_datetime", + "convert_datetime_to_iso", + "convert_mactime_to_datetime", + "convert_mactime_to_iso", + "convert_unix_to_iso", + "convert_unix_to_utc_datetime", + # MVT's version. + "MVT_VERSION", +] diff --git a/tests/android/test_artifact_dumpsys_accessibility.py b/tests/android/test_artifact_dumpsys_accessibility.py index 1f0b234..c727c57 100644 --- a/tests/android/test_artifact_dumpsys_accessibility.py +++ b/tests/android/test_artifact_dumpsys_accessibility.py @@ -23,7 +23,7 @@ class TestDumpsysAccessibilityArtifact: assert len(da.results) == 4 assert da.results[0]["package_name"] == "com.android.settings" assert ( - da.results[0]["service"] + da.results[0]["component"] == "com.android.settings/com.samsung.android.settings.development.gpuwatch.GPUWatchInterceptor" ) @@ -37,7 +37,9 @@ class TestDumpsysAccessibilityArtifact: da.parse(data) assert len(da.results) == 1 assert da.results[0]["package_name"] == "com.malware.accessibility" - assert da.results[0]["service"] == "com.malware.service.malwareservice" + assert da.results[0]["service_name"] == "com.malware.service.malwareservice" + assert da.results[0]["enabled"] is True + assert da.results[0]["installed"] is False def test_accessibility_service_alert(self): da = DumpsysAccessibilityArtifact() @@ -52,6 +54,22 @@ class TestDumpsysAccessibilityArtifact: assert da.alertstore.alerts[0].level == AlertLevel.MEDIUM assert da.alertstore.alerts[0].event == da.results[0] + def test_same_component_is_kept_for_each_user(self): + da = DumpsysAccessibilityArtifact() + da.parse( + """User state[attributes:{id=0 + installed services: { + 0 : com.example/.Service + } +User state[attributes:{id=10 + installed services: { + 0 : com.example/.Service + } +""" + ) + + assert [result["user_id"] for result in da.results] == [0, 10] + def test_ioc_check(self, indicator_file): da = DumpsysAccessibilityArtifact() file = get_artifact("android_data/dumpsys_accessibility.txt") diff --git a/tests/android/test_artifact_dumpsys_adb.py b/tests/android/test_artifact_dumpsys_adb.py index f5574a6..8d501b7 100644 --- a/tests/android/test_artifact_dumpsys_adb.py +++ b/tests/android/test_artifact_dumpsys_adb.py @@ -3,6 +3,8 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ +import base64 + from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact from mvt.android.modules.bugreport.dumpsys_adb_state import DumpsysADBState from mvt.common.alerts import AlertLevel @@ -11,6 +13,17 @@ from ..utils import get_artifact class TestDumpsysADBArtifact: + def test_parsing_binary_xml_recovers_key(self): + public_key = base64.b64encode(bytes(range(256))) + keystore = DumpsysADBArtifact().parse_binary_xml( + b"ABX\x00binary-key=" + public_key + b" user@host\x00lastConnection" + ) + + assert len(keystore) == 1 + assert keystore[0]["key"] == public_key.decode() + assert keystore[0]["user"] == "user@host" + assert keystore[0]["last_connected"] is None + def test_parsing(self): da_adb = DumpsysADBArtifact() file = get_artifact("android_data/dumpsys_adb.txt") diff --git a/tests/android/test_artifact_dumpsys_appops.py b/tests/android/test_artifact_dumpsys_appops.py index 862b872..a86084f 100644 --- a/tests/android/test_artifact_dumpsys_appops.py +++ b/tests/android/test_artifact_dumpsys_appops.py @@ -25,10 +25,37 @@ class TestDumpsysAppopsArtifact: assert da.results[0]["uid"] == "0" assert len(da.results[0]["permissions"]) == 1 assert da.results[0]["permissions"][0]["name"] == "MANAGE_IPSEC_TUNNELS" - assert da.results[0]["permissions"][0]["access"] == "allow" + assert da.results[0]["permissions"][0]["mode"] == "allow" assert da.results[6]["package_name"] == "com.sec.factory.camera" assert len(da.results[6]["permissions"][1]["entries"]) == 1 assert len(da.results[11]["permissions"]) == 4 + wake_lock = next( + permission + for permission in da.results[11]["permissions"] + if permission["name"] == "WAKE_LOCK" + ) + assert wake_lock["entries"][0]["duration"] == "+126ms" + + def test_running_and_attribution_are_retained(self): + da = DumpsysAppopsArtifact() + da.parse( + """ Uid 0: + state=cch + Package com.example: + CAMERA (allow): + camera=[ + Access: [fg-s] 2025-01-01 00:00:00.000 (-1s) duration=+2ms + ] + RECORD_AUDIO (allow): + Running start at: +3s +""" + ) + + camera = da.results[0]["permissions"][0]["entries"][0] + running = da.results[0]["permissions"][1]["entries"][0] + assert camera["attribution"] == "camera" + assert running["event"] == "running" + assert running["relative_time"] == "+3s" def test_ioc_check(self, indicator_file): da = DumpsysAppopsArtifact() diff --git a/tests/android/test_artifact_dumpsys_battery_daily.py b/tests/android/test_artifact_dumpsys_battery_daily.py index dbb9c36..5f7f997 100644 --- a/tests/android/test_artifact_dumpsys_battery_daily.py +++ b/tests/android/test_artifact_dumpsys_battery_daily.py @@ -57,18 +57,19 @@ class TestDumpsysBatteryDailyArtifact: assert uninstall_alert.message == ( "Detected uninstall of package com.example.removed (vers 0)" ) - assert uninstall_alert.event_time == "2022-08-16" + assert uninstall_alert.event_time == "2022-08-16 15:56:39" assert uninstall_alert.event["package_name"] == "com.example.removed" - assert uninstall_alert.event["vers"] == "0" + assert uninstall_alert.event["version_code"] == 0 + assert uninstall_alert.event["action"] == "uninstall" assert downgrade_alert.level == AlertLevel.MEDIUM assert downgrade_alert.message == ( "Detected downgrade of package com.example.app from vers 10 to vers 9" ) - assert downgrade_alert.event_time == "2022-08-17" + assert downgrade_alert.event_time == "2022-08-17 15:56:39" assert downgrade_alert.event["package_name"] == "com.example.app" assert downgrade_alert.event["action"] == "downgrade" - assert downgrade_alert.event["previous_vers"] == "10" + assert downgrade_alert.event["previous_version_code"] == 10 def test_newest_first_update_is_not_reported_as_downgrade(self): dba = DumpsysBatteryDailyArtifact() @@ -107,7 +108,19 @@ class TestDumpsysBatteryDailyArtifact: assert downgrade_alert.event_time == "2026-01-10" assert downgrade_alert.event["package_name"] == "com.example.app" assert downgrade_alert.event["action"] == "downgrade" - assert downgrade_alert.event["previous_vers"] == "102" + assert downgrade_alert.event["previous_version_code"] == 102 + + def test_duplicate_updates_retain_occurrence_count(self): + dba = DumpsysBatteryDailyArtifact() + dba.parse( + """ Daily from 2026-01-10-01-02-03 to 2026-01-11-04-05-06: + Update com.example.app vers=12 + Update com.example.app vers=12 +""" + ) + + assert dba.results[0]["occurrences"] == 2 + assert dba.results[0]["period_start"] == "2026-01-10 01:02:03" def test_reinstall_after_uninstall_is_not_reported_as_downgrade(self): dba = DumpsysBatteryDailyArtifact() diff --git a/tests/android/test_artifact_dumpsys_battery_history.py b/tests/android/test_artifact_dumpsys_battery_history.py index dd0b120..091ce55 100644 --- a/tests/android/test_artifact_dumpsys_battery_history.py +++ b/tests/android/test_artifact_dumpsys_battery_history.py @@ -54,6 +54,7 @@ class TestDumpsysBatteryHistoryArtifact: assert len(dba.results) == 2 assert dba.results[0] == { "time_elapsed": "07-15 20:27:39.431", + "timestamp": "1900-07-15 20:27:39.431000", "event": "start_job", "uid": "u0a123", "package_name": "com.example", @@ -61,3 +62,23 @@ class TestDumpsysBatteryHistoryArtifact: } assert dba.results[1]["event"] == "end_job" assert dba.results[1]["uid"] == "u0a123" + + def test_wake_lock_without_component_is_retained(self): + dba = DumpsysBatteryHistoryArtifact() + dba.parse( + "Battery History:\n" + " 0 (2) 100 RESET:TIME: 2025-09-05-01-04-52-139\n" + ' +1s (2) 100 +running +wake_lock=1000:"*alarm*:TIME_TICK"\n' + ' +2s (2) 100 +running +wake_lock=u0a1:"*walarm*:com.whatsapp.MessageHandler.LOGOUT_ACTION"\n' + "\n" + ) + + assert [record["event"] for record in dba.results] == ["wake", "wake"] + assert dba.results[0]["package_name"] is None + assert dba.results[1]["package_name"] == "com.whatsapp" + + def test_decorated_sync_job_uses_component_package(self): + dba = DumpsysBatteryHistoryArtifact() + dba.parse('+1s (2) 100 +job=u0a1:"@SyncManager@gmail-ls/com.google:android"\n') + + assert dba.results[0]["package_name"] == "com.google" diff --git a/tests/android/test_artifact_dumpsys_dbinfo.py b/tests/android/test_artifact_dumpsys_dbinfo.py index 691d43c..0dcc0bc 100644 --- a/tests/android/test_artifact_dumpsys_dbinfo.py +++ b/tests/android/test_artifact_dumpsys_dbinfo.py @@ -53,9 +53,32 @@ Connection pool for /data/user/0/com.example/databases/current.db: assert dbi.results == [ { - "isodate": "07-15 20:27:39.431", + "timestamp": "07-15 20:27:39.431", + "pid": None, "action": "executeForCursorWindow", + "duration_ms": 1, + "status": "succeeded", "sql": "SELECT 1", "path": "/data/user/0/com.example/databases/current.db", + "pool_path": "/data/user/0/com.example/databases/current.db", + "connection_number": None, + "is_primary": None, } ] + + def test_parses_operations_from_multiple_connections(self): + dbi = DumpsysDBInfoArtifact() + dbi.parse( + """Connection pool for /data/example.db: + Connection #0: + isPrimaryConnection: true + Most recently executed operations: + 0: [2025-01-01 00:00:00.000] execute took 1ms - succeeded, sql="SELECT 1", path=/data/example.db + Connection #1: + isPrimaryConnection: false + Most recently executed operations: + 0: [2025-01-01 00:00:01.000] execute took 2ms - succeeded, sql="SELECT 2", path=/data/example.db +""" + ) + + assert [record["connection_number"] for record in dbi.results] == [0, 1] diff --git a/tests/android/test_artifact_dumpsys_package_activities.py b/tests/android/test_artifact_dumpsys_package_activities.py index 5eab63d..66ef790 100644 --- a/tests/android/test_artifact_dumpsys_package_activities.py +++ b/tests/android/test_artifact_dumpsys_package_activities.py @@ -21,12 +21,16 @@ class TestDumpsysPackageActivitiesArtifact: assert len(dpa.results) == 0 dpa.parse(data) - assert len(dpa.results) == 4 - assert dpa.results[0]["package_name"] == "com.samsung.android.app.social" + assert len(dpa.results) == 10 + assert dpa.results[0]["package_name"] == "com.samsung.android.messaging" assert ( - dpa.results[0]["activity"] - == "com.samsung.android.app.social/.feed.FeedsActivity" + dpa.results[0]["component"] + == "com.samsung.android.messaging/.ui.RcsTransferContent" ) + assert {result["resolver_type"] for result in dpa.results} == { + "full_mime_type", + "non_data_action", + } def test_ioc_check(self, indicator_file): dpa = DumpsysPackageActivitiesArtifact() @@ -41,4 +45,4 @@ class TestDumpsysPackageActivitiesArtifact: dpa.indicators = ind assert len(dpa.alertstore.alerts) == 0 dpa.check_indicators() - assert len(dpa.alertstore.alerts) == 1 + assert len(dpa.alertstore.alerts) == 2 diff --git a/tests/android/test_artifact_dumpsys_packages.py b/tests/android/test_artifact_dumpsys_packages.py index bd5fb75..f22cc67 100644 --- a/tests/android/test_artifact_dumpsys_packages.py +++ b/tests/android/test_artifact_dumpsys_packages.py @@ -25,7 +25,11 @@ class TestDumpsysPackagesArtifact: == "com.samsung.android.provider.filterprovider" ) assert dpa.results[0]["version_name"] == "5.0.07" - assert dpa.results[0]["first_install_time"] == "2008-12-31 16:00:00" + assert dpa.results[0]["version_code"] == 500700000 + assert dpa.results[0]["min_sdk"] == 28 + assert dpa.results[0]["target_sdk"] == 28 + assert dpa.results[0]["package_type"] == "active" + assert dpa.results[0]["users"][0]["user_id"] == 0 assert dpa.results[0]["system"] is True def test_parsing_system_flag(self): @@ -60,42 +64,27 @@ class TestDumpsysPackagesArtifact: dpa.check_indicators() assert len(dpa.alertstore.alerts) == 1 - def test_per_user_fields_use_primary_user(self): - details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details( - """ User 0: installed=true - firstInstallTime=2024-01-10 09:19:39 - runtime permissions: - android.permission.CAMERA: granted=true - User 95: installed=false - firstInstallTime=1970-01-01 01:00:00 - runtime permissions: - android.permission.CAMERA: granted=false - android.permission.RECORD_AUDIO: granted=false + def test_hidden_packages_and_per_user_state(self): + dpa = DumpsysPackagesArtifact() + dpa.parse( + """Packages: + Package [com.example.active]: + appId=10001 + versionCode=12 minSdk=29 targetSdk=35 + User 0: installed=true hidden=false + firstInstallTime=2025-01-01 01:02:03 + User 10: installed=false hidden=true + firstInstallTime=2025-01-02 01:02:03 +Hidden system packages: + Package [com.example.hidden]: + appId=10002 + installerPackageName=null """ ) - assert details["first_install_time"] == "2024-01-10 09:19:39" - runtime_permissions = [ - permission - for permission in details["permissions"] - if permission["type"] == "runtime" + assert [record["package_type"] for record in dpa.results] == [ + "active", + "hidden_system", ] - assert runtime_permissions == [ - { - "name": "android.permission.CAMERA", - "granted": True, - "type": "runtime", - } - ] - - def test_per_user_fields_fall_back_when_user_zero_is_missing(self): - details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details( - """ User 10: installed=true - firstInstallTime=2024-02-10 09:19:39 - runtime permissions: - android.permission.CAMERA: granted=true -""" - ) - - assert details["first_install_time"] == "2024-02-10 09:19:39" - assert details["permissions"][-1]["name"] == "android.permission.CAMERA" + assert len(dpa.results[0]["users"]) == 2 + assert dpa.results[1]["installer"] is None diff --git a/tests/android/test_artifact_dumpsys_platform_compat.py b/tests/android/test_artifact_dumpsys_platform_compat.py index c8c59b3..c202645 100644 --- a/tests/android/test_artifact_dumpsys_platform_compat.py +++ b/tests/android/test_artifact_dumpsys_platform_compat.py @@ -22,6 +22,9 @@ class TestDumpsysPlatformCompatArtifact: assert len(dbi.results) == 2 assert dbi.results[0]["package_name"] == "org.torproject.torbrowser" assert dbi.results[1]["package_name"] == "org.article19.circulo.next" + assert dbi.results[0]["change_id"] == 168419799 + assert dbi.results[0]["change_name"] == "DOWNSCALED" + assert dbi.results[0]["override_value"] is False def test_ioc_check(self, indicator_file): dbi = DumpsysPlatformCompatArtifact() diff --git a/tests/android/test_artifact_dumpsys_receivers.py b/tests/android/test_artifact_dumpsys_receivers.py index 7875a52..6e8b92e 100644 --- a/tests/android/test_artifact_dumpsys_receivers.py +++ b/tests/android/test_artifact_dumpsys_receivers.py @@ -19,17 +19,14 @@ class TestDumpsysReceiversArtifact: assert len(dr.results) == 0 dr.parse(data) - assert len(dr.results) == 4 - assert ( - list(dr.results.keys())[0] - == "com.android.storagemanager.automatic.SHOW_NOTIFICATION" - ) - assert ( - dr.results["com.android.storagemanager.automatic.SHOW_NOTIFICATION"][0][ - "package_name" - ] - == "com.android.storagemanager" + assert len(dr.results) == 9 + assert dr.results[0]["resolver_type"] == "full_mime_type" + storage_manager = next( + result + for result in dr.results + if result["key"] == "com.android.storagemanager.automatic.SHOW_NOTIFICATION" ) + assert storage_manager["package_name"] == "com.android.storagemanager" def test_parsing_misindented_action(self): dr = DumpsysReceiversArtifact() @@ -44,12 +41,8 @@ Receiver Resolver Table: dr.parse(data) - assert ( - dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][ - "package_name" - ] - == "com.psycatgames.nhiegame" - ) + assert dr.results[1]["key"] == "android.intent.action.MY_PACKAGE_REPLACED" + assert dr.results[1]["package_name"] == "com.psycatgames.nhiegame" def test_parsing_misindented_first_action(self): dr = DumpsysReceiversArtifact() @@ -62,12 +55,8 @@ Receiver Resolver Table: dr.parse(data) - assert ( - dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][ - "package_name" - ] - == "com.psycatgames.nhiegame" - ) + assert dr.results[0]["key"] == "android.intent.action.MY_PACKAGE_REPLACED" + assert dr.results[0]["package_name"] == "com.psycatgames.nhiegame" def test_ioc_check(self, indicator_file): dr = DumpsysReceiversArtifact() diff --git a/tests/android/test_artifact_getprop.py b/tests/android/test_artifact_getprop.py index d9fec6b..0fbe742 100644 --- a/tests/android/test_artifact_getprop.py +++ b/tests/android/test_artifact_getprop.py @@ -39,3 +39,29 @@ class TestGetPropArtifact: assert len(gp.alertstore.alerts) == 0 gp.check_indicators() assert len(gp.alertstore.alerts) == 1 + + def test_empty_values_and_invalid_lines(self): + gp = GetProp() + gp.parse("[empty]: []\n[valid]: [value]\n0\n[broken]: [value") + + assert gp.results == [ + {"name": "empty", "value": ""}, + {"name": "valid", "value": "value"}, + ] + + def test_multiline_value(self): + gp = GetProp() + gp.parse( + "[persist.sys.boot.reason.history]: [" + "reboot,ota,1697044974\n" + "reboot,watchdog,1696958574]\n" + "[ro.build.version.sdk]: [35]\n" + ) + + assert gp.results == [ + { + "name": "persist.sys.boot.reason.history", + "value": "reboot,ota,1697044974\nreboot,watchdog,1696958574", + }, + {"name": "ro.build.version.sdk", "value": "35"}, + ] diff --git a/tests/android/test_artifact_processes.py b/tests/android/test_artifact_processes.py index 2806472..4d51037 100644 --- a/tests/android/test_artifact_processes.py +++ b/tests/android/test_artifact_processes.py @@ -20,7 +20,7 @@ class TestProcessesArtifact: assert len(p.results) == 0 p.parse(data) assert len(p.results) == 17 - assert p.results[0]["proc_name"] == "init" + assert p.results[0]["command"] == "init" def test_ioc_check(self, indicator_file): p = Processes() @@ -36,3 +36,14 @@ class TestProcessesArtifact: assert len(p.alertstore.alerts) == 0 p.check_indicators() assert len(p.alertstore.alerts) == 1 + + def test_bugreport_thread_columns(self): + p = Processes() + p.parse( + "LABEL USER PID TID PPID VSZ RSS WCHAN ADDR S PRI NI RTPRIO SCH PCY TIME CMD\n" + "u:r:init:s0 root 1 2 0 100 20 0 0 S 19 0 - 0 fg 00:00:01 init\n" + ) + + assert p.results[0]["label"] == "u:r:init:s0" + assert p.results[0]["tid"] == 2 + assert p.results[0]["command"] == "init" diff --git a/tests/android/test_artifact_tombstone_caused_by.py b/tests/android/test_artifact_tombstone_caused_by.py new file mode 100644 index 0000000..9a2df85 --- /dev/null +++ b/tests/android/test_artifact_tombstone_caused_by.py @@ -0,0 +1,54 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +"""A `Caused by:` line must not discard the whole text tombstone. + +Keys are matched as bare prefixes, so `Caused by: …` — an ordinary line inside +an abort message — reached the `Cause` key, failed the key comparison and +raised, which `Tombstones.run()` logged while dropping the entire crash record. +Seen on a 1.6 MB tombstone whose protobuf twin was zero bytes: the crash then +had no representation at all. +""" + +import datetime + +from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact + +TOMBSTONE = b"""\ +*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +Build fingerprint: 'Xiaomi/vili_eea/vili:13/TKQ1.220829.002/V14.0.10.0:user/release-keys' +Revision: '0' +ABI: 'arm64' +Timestamp: 2023-08-24 14:54:47.999124034+0300 +Process uptime: 12199s +Cmdline: com.example.game +pid: 8044, tid: 26222, name: UnityMain >>> com.example.game <<< +uid: 10235 +signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr -------- +Abort message: 'No pending exception expected: java.lang.SecurityException: listen + at void android.os.Parcel.readException() (Parcel.java:2920) +Caused by: android.os.RemoteException: Remote stack trace: +\tat com.android.server.TelephonyRegistry.listen(TelephonyRegistry.java:1096) +""" + +WITH_CAUSE = TOMBSTONE + b"Cause: null pointer dereference\n" + + +class TestTombstoneCausedBy: + def _parse(self, content): + artifact = TombstoneCrashArtifact() + artifact.results = [] + artifact.parse("tombstone_23", datetime.datetime(2023, 8, 24), content) + return artifact.results + + def test_caused_by_line_does_not_discard_the_tombstone(self): + results = self._parse(TOMBSTONE) + assert len(results) == 1 + assert results[0]["pid"] == 8044 + assert results[0]["process_name"] == "UnityMain" + assert results[0]["uid"] == 10235 + + def test_the_real_cause_key_is_still_parsed(self): + results = self._parse(WITH_CAUSE) + assert results[0]["cause"] == "null pointer dereference" diff --git a/tests/android/test_artifact_tombstones.py b/tests/android/test_artifact_tombstones.py index f88a9c2..e9159bd 100644 --- a/tests/android/test_artifact_tombstones.py +++ b/tests/android/test_artifact_tombstones.py @@ -128,8 +128,5 @@ class TestTombstoneCrashArtifact: assert tombstone_result.get("pid") == 25541 assert tombstone_result.get("process_name") == "mtk.ape.decoder" - # With Android logs we want to keep timestamps as device local time for consistency. - # We often don't know the time offset for a log entry and so can't convert everything to UTC. - # MVT should output the local time only: - # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000 - assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.518290" + # Tombstones include an explicit offset, so normalize them to UTC. + assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.518290" diff --git a/tests/android/test_check_backup_optional_failure.py b/tests/android/test_check_backup_optional_failure.py new file mode 100644 index 0000000..72dae42 --- /dev/null +++ b/tests/android/test_check_backup_optional_failure.py @@ -0,0 +1,39 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +"""An encrypted backup.ab must not take the whole check-androidqf run with it. + +`CmdAndroidCheckBackup.from_ab()` already raises `InvalidAndroidBackup` instead +of exiting when it runs as a sub-command (`check-androidqf` catches that and +skips the backup modules), for a wrong file format and for a parse error. The +password branches used to call `sys.exit(1)` unconditionally, which ends the +parent run inside `finish()` — before the intrusion-logs command and before the +timeline, alerts, urls, info and run-manifest are written. +""" + +import pytest + +from mvt.android.cmd_check_backup import CmdAndroidCheckBackup, InvalidAndroidBackup + +ENCRYPTED_AB_HEADER = b"ANDROID BACKUP\n5\n0\nAES-256\n" + b"\x00" * 64 + + +class TestCheckBackupOptionalFailure: + def _cmd(self, tmp_path, sub_command): + return CmdAndroidCheckBackup( + target_path=None, + results_path=str(tmp_path), + module_options={"interactive": False}, + sub_command=sub_command, + ) + + def test_missing_password_raises_when_nested(self, tmp_path): + cmd = self._cmd(tmp_path, sub_command=True) + with pytest.raises(InvalidAndroidBackup): + cmd.from_ab(ENCRYPTED_AB_HEADER) + + def test_missing_password_still_exits_on_its_own_command(self, tmp_path): + cmd = self._cmd(tmp_path, sub_command=False) + with pytest.raises(SystemExit): + cmd.from_ab(ENCRYPTED_AB_HEADER) diff --git a/tests/android_androidqf/test_mounts.py b/tests/android_androidqf/test_mounts.py index d66925d..5f47176 100644 --- a/tests/android_androidqf/test_mounts.py +++ b/tests/android_androidqf/test_mounts.py @@ -13,6 +13,19 @@ from ..utils import get_android_androidqf, list_files class TestAndroidqfMountsArtifact: + def test_parse_proc_mountinfo(self): + from mvt.android.artifacts.mounts import Mounts as MountsArtifact + + results = MountsArtifact.parse_mountinfo( + "41 40 254:13 / / ro,relatime shared:1 - erofs /dev/block/dm-13 ro,seclabel\n", + 123, + ) + + assert results[0]["mount_id"] == 41 + assert results[0]["mount_point"] == "/" + assert results[0]["filesystem_type"] == "erofs" + assert results[0]["process_ids"] == [123] + def test_parse_mounts_token_checks(self): """ Test the artifact-level `parse` method using tolerant token checks. diff --git a/tests/android_androidqf/test_settings.py b/tests/android_androidqf/test_settings.py index ce14460..6edfec8 100644 --- a/tests/android_androidqf/test_settings.py +++ b/tests/android_androidqf/test_settings.py @@ -6,12 +6,27 @@ from pathlib import Path from mvt.android.modules.androidqf.aqf_settings import AQFSettings +from mvt.android.artifacts.settings import Settings from mvt.common.module import run_module from ..utils import get_android_androidqf, list_files class TestSettingsModule: + def test_bugreport_settings_format(self): + settings = Settings() + settings.parse( + "GLOBAL SETTINGS (user 0)\n" + "_id:1 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true\n" + "SECURE SETTINGS (user 10)\n" + "_id:2 name:accessibility_enabled pkg:android value:1\n" + ) + + assert settings.results == { + "global:user_0": {"adb_wifi_enabled": "0"}, + "secure:user_10": {"accessibility_enabled": "1"}, + } + def test_parsing(self): data_path = get_android_androidqf() m = AQFSettings(target_path=data_path) diff --git a/tests/android_bugreport/test_bugreport.py b/tests/android_bugreport/test_bugreport.py index 75df106..413bb18 100644 --- a/tests/android_bugreport/test_bugreport.py +++ b/tests/android_bugreport/test_bugreport.py @@ -54,10 +54,11 @@ class TestBugreportAnalysis: == "com.samsung.android.provider.filterprovider" ) assert m.results[1]["package_name"] == "com.instagram.android" - assert m.results[0]["installer"] == "" + assert m.results[0]["installer"] is None assert m.results[1]["installer"] == "com.android.vending" assert len(m.results[0]["permissions"]) == 4 - assert len(m.results[1]["permissions"]) == 32 + assert len(m.results[1]["permissions"]) == 20 + assert len(m.results[1]["users"][0]["permissions"]) == 19 def test_getprop_module(self): m = self.launch_bug_report_module(DumpsysGetProp) @@ -66,29 +67,34 @@ class TestBugreportAnalysis: def test_receivers_match_exact_package_name(self, indicators_factory): intent = "android.intent.action.PHONE_STATE" false_positive = { + "resolver_type": "non_data_action", + "key": intent, "package_name": "com.android.phone", - "receiver": ( + "component": ( "com.android.phone/" "com.android.services.telephony.sip.SipIncomingCallReceiver" ), + "filter_count": 1, } malicious_receiver = { + "resolver_type": "non_data_action", + "key": intent, "package_name": "com.android.services", - "receiver": "com.android.services/com.example.SomeReceiver", + "component": "com.android.services/com.example.SomeReceiver", + "filter_count": 1, } - module = DumpsysReceivers( - results={intent: [false_positive, malicious_receiver]} - ) + module = DumpsysReceivers(results=[false_positive, malicious_receiver]) module.indicators = indicators_factory(app_ids=["com.android.services"]) module.check_indicators() assert len(module.alertstore.alerts) == 1 alert = module.alertstore.alerts[0] - assert alert.event == {intent: malicious_receiver} + assert alert.event == malicious_receiver assert alert.matched_indicator.value == "com.android.services" def test_tombstones_modules(self): m = self.launch_bug_report_module(Tombstones) assert len(m.results) == 2 assert m.results[1]["pid"] == 3559 + assert m.results[0]["sources"]["text"]["parsed"] is True diff --git a/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee b/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee new file mode 100644 index 0000000..1d27832 Binary files /dev/null and b/tests/artifacts/ios_backup/1f/1f5a521220a3ad80ebfdc196978df8e7a2e49dee differ diff --git a/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d b/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d new file mode 100644 index 0000000..b8ae144 Binary files /dev/null and b/tests/artifacts/ios_backup/7c/7c7fba66680ef796b916b067077cc246adacf01d differ diff --git a/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f b/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f new file mode 100644 index 0000000..132059c Binary files /dev/null and b/tests/artifacts/ios_backup/b8/b8548dc30aa1030df0ce18ef08b882cf7ab5212f differ diff --git a/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 b/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 new file mode 100644 index 0000000..d54a9c5 Binary files /dev/null and b/tests/artifacts/ios_backup/e7/e794f6ffcc3c222535f47684a63d5178da3c4500 differ diff --git a/tests/common/test_cli_plugins.py b/tests/common/test_cli_plugins.py index ea7c7fa..5abb2c9 100644 --- a/tests/common/test_cli_plugins.py +++ b/tests/common/test_cli_plugins.py @@ -3,9 +3,12 @@ from types import SimpleNamespace import click from click.testing import CliRunner +from mvt.cli import cli as mvt_cli from mvt.common.cli_plugins import ( ANDROID_CLI_PLUGIN_GROUP, IOS_CLI_PLUGIN_GROUP, + MVT_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, BrokenPluginCommand, load_cli_commands_option, register_cli_commands_from_path, @@ -14,6 +17,9 @@ from mvt.common.cli_plugins import ( ) +# Keep the banner of the mvt group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + COMMAND_TEMPLATE = """ import click @@ -345,7 +351,11 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate( def entry_points(*, group): if group == IOS_CLI_PLUGIN_GROUP: return [_entry_point("ios-package", "ios_plugin:cli", ios_package)] - return [_entry_point("android-package", "android_plugin:cli", android_package)] + if group == ANDROID_CLI_PLUGIN_GROUP: + return [ + _entry_point("android-package", "android_plugin:cli", android_package) + ] + return [] monkeypatch.setattr( "mvt.common.cli_plugins.importlib.metadata.entry_points", @@ -369,3 +379,187 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate( assert set(ios_group.commands) == {"ios-file", "ios-package"} assert set(android_group.commands) == {"android-file", "android-package"} + + +def test_neutral_entry_point_group_is_not_registered_on_the_platform_clis( + monkeypatch, +): + @click.command() + def neutral_package(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point("neutral-package", "neutral_plugin:cli", neutral_package) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + ios_group = click.Group() + android_group = click.Group() + + register_cli_plugins( + ios_group, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable="TEST_IOS_COMMANDS", + ) + register_cli_plugins( + android_group, + entry_point_group=ANDROID_CLI_PLUGIN_GROUP, + environment_variable="TEST_ANDROID_COMMANDS", + ) + + assert not ios_group.commands + assert not android_group.commands + + +def test_environment_command_wins_collision_with_installed_command( + tmp_path, monkeypatch, caplog +): + command_path = _write_command( + tmp_path / "duplicate.py", + "duplicate", + message="environment command ran", + ) + + @click.command() + def installed_command(): + pass + + def entry_points(*, group): + if group == IOS_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "duplicate", + "ios_plugin:cli", + installed_command, + distribution="ios-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + monkeypatch.setenv("TEST_IOS_COMMANDS", str(command_path)) + group = click.Group() + + register_cli_plugins( + group, + entry_point_group=IOS_CLI_PLUGIN_GROUP, + environment_variable="TEST_IOS_COMMANDS", + ) + + assert group.commands["duplicate"] is not installed_command + result = CliRunner().invoke(group, ["duplicate"]) + assert result.exit_code == 0 + assert "environment command ran" in result.output + assert "the command name is already registered" in caplog.text + assert "ios-plugin 1.0 (ios_plugin:cli)" in caplog.text + + +def test_the_mvt_cli_gets_the_neutral_commands_and_no_platform_command( + monkeypatch, restore_cli_commands +): + @click.command() + def shared_package(): + click.echo("shared command ran") + + @click.command() + def ios_package(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [_entry_point("shared-package", "shared_plugin:cli", shared_package)] + if group == IOS_CLI_PLUGIN_GROUP: + return [_entry_point("ios-package", "ios_plugin:cli", ios_package)] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert "ios-package" not in mvt_cli.commands + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "shared-package"]) + assert result.exit_code == 0 + assert "shared command ran" in result.output + + +def test_builtin_mvt_command_wins_collision_with_neutral_command( + monkeypatch, caplog, restore_cli_commands +): + @click.command() + def neutral_version(): + pass + + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "version", + "neutral_plugin:cli", + neutral_version, + distribution="neutral-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + builtin_version = mvt_cli.commands["version"] + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert mvt_cli.commands["version"] is builtin_version + assert "the command name is already registered" in caplog.text + assert "neutral-plugin 1.0 (neutral_plugin:cli)" in caplog.text + + +def test_broken_neutral_plugin_does_not_break_the_mvt_cli( + monkeypatch, restore_cli_commands +): + def entry_points(*, group): + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [ + _entry_point( + "broken", + "broken_plugin:cli", + exception=RuntimeError("missing dependency"), + distribution="broken-plugin", + ) + ] + return [] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", + entry_points, + ) + + register_cli_plugins( + mvt_cli, + entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP, + environment_variable=MVT_CUSTOM_COMMANDS_ENV, + ) + + assert isinstance(mvt_cli.commands["broken"], BrokenPluginCommand) + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "version"]) + assert result.exit_code == 0 diff --git a/tests/common/test_cmd_check_iocs.py b/tests/common/test_cmd_check_iocs.py new file mode 100644 index 0000000..0c05c1b --- /dev/null +++ b/tests/common/test_cmd_check_iocs.py @@ -0,0 +1,225 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import json +import logging + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES +from mvt.common.cmd_check_iocs import CmdCheckIOCS +from mvt.common.module import MVTModule +from mvt.ios.cli import cli as ios_cli +from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES +from mvt.ios.modules.backup.manifest import Manifest + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + + +class CustomResultsModule(MVTModule): + """A custom module which declares the check-iocs pair of both platforms.""" + + slug = "custom_results" + supported_commands = ( + ("ios", "check-backup"), + ("ios", "check-iocs"), + ("android", "check-iocs"), + ) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BackupCheckerModule(MVTModule): + """An iOS module which implements check_indicators() without declaring check-iocs.""" + + slug = "backup_checker" + supported_commands = (("ios", "check-backup"),) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BugReportCheckerModule(MVTModule): + """The same for Android.""" + + slug = "bugreport_checker" + supported_commands = (("android", "check-bugreport"),) + + checked: list = [] + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.checked.append(list(self.results)) + + +class BackupOnlyModule(MVTModule): + """A custom module which does not implement check_indicators().""" + + slug = "backup_only" + supported_commands = (("ios", "check-backup"),) + + def run(self) -> None: + pass + + +@pytest.mark.parametrize( + "platform, builtin_modules, checker_module", + [ + ("ios", IOS_CHECK_IOCS_MODULES, BackupCheckerModule), + ("android", ANDROID_CHECK_IOCS_MODULES, BugReportCheckerModule), + ], +) +def test_check_iocs_rechecks_the_stored_results_of_custom_modules( + platform, builtin_modules, checker_module, tmp_path, caplog +): + # check-iocs matches every .json in the results folder to the module + # with that slug, custom modules included, and runs its check_indicators() + # again over the stored results. + results = [{"domain": "example.org"}] + (tmp_path / "custom_results.json").write_text(json.dumps(results)) + (tmp_path / f"{checker_module.slug}.json").write_text(json.dumps(results)) + (tmp_path / "backup_only.json").write_text(json.dumps(results)) + CustomResultsModule.checked.clear() + checker_module.checked.clear() + + cmd = CmdCheckIOCS( + target_path=str(tmp_path), + custom_modules=[CustomResultsModule, checker_module, BackupOnlyModule], + platform=platform, + ) + cmd.modules = builtin_modules + + with caplog.at_level(logging.INFO): + cmd.run() + + # A module which declares the check-iocs pair is re-checked. + assert CustomResultsModule.checked == [results] + assert ( + 'Loading results from "custom_results.json" with module CustomResultsModule' + in caplog.text + ) + # So is a module which only implements check_indicators(). + assert checker_module.checked == [results] + # A module which does neither is not part of check-iocs. + assert "backup_only.json" not in caplog.text + + +@pytest.mark.parametrize( + "platform, builtin_modules, listed, not_listed", + [ + ( + "ios", + IOS_CHECK_IOCS_MODULES, + "BackupCheckerModule", + "BugReportCheckerModule", + ), + ( + "android", + ANDROID_CHECK_IOCS_MODULES, + "BugReportCheckerModule", + "BackupCheckerModule", + ), + ], +) +def test_check_iocs_lists_the_custom_modules_it_runs( + platform, builtin_modules, listed, not_listed, caplog +): + cmd = CmdCheckIOCS( + custom_modules=[ + CustomResultsModule, + BackupCheckerModule, + BugReportCheckerModule, + BackupOnlyModule, + ], + platform=platform, + ) + cmd.modules = builtin_modules + + with caplog.at_level(logging.INFO): + cmd.list_modules() + + assert "CustomResultsModule" in caplog.text + # The module which implements check_indicators() for this platform is listed. + assert listed in caplog.text + # The one for the other platform is not, and neither is BackupOnlyModule. + assert not_listed not in caplog.text + assert "BackupOnlyModule" not in caplog.text + + +class ReplacementManifest(Manifest): + """A replacement for a built-in module which does not declare check-iocs.""" + + supported_commands = (("ios", "check-backup"),) + replaces = Manifest + + +def test_check_iocs_uses_a_replacement_of_a_built_in_module(): + # A replacement which subclasses a built-in module inherits its + # check_indicators(). check-iocs then runs it in place of that module. + cmd = CmdCheckIOCS(custom_modules=[ReplacementManifest], platform="ios") + cmd.modules = IOS_CHECK_IOCS_MODULES + + available = cmd._available_modules() + + assert ReplacementManifest in available + assert Manifest not in available + + +LOADED_MODULE = ''' +from mvt.common.module import MVTModule + + +class LoadedResultsModule(MVTModule): + """A module loaded from a file with --load-module.""" + + slug = "loaded_results" + supported_commands = (("ios", "check-iocs"), ("android", "check-iocs")) + + def run(self) -> None: + pass + + def check_indicators(self) -> None: + self.log.warning("loaded module checked %d results", len(self.results)) +''' + + +@pytest.mark.parametrize("cli", [ios_cli, android_cli], ids=["mvt-ios", "mvt-android"]) +def test_check_iocs_loads_custom_modules_from_a_file_on_each_cli(cli, tmp_path, caplog): + module_path = tmp_path / "loaded_module.py" + module_path.write_text(LOADED_MODULE) + results_folder = tmp_path / "results" + results_folder.mkdir() + (results_folder / "loaded_results.json").write_text(json.dumps([{"a": 1}])) + + with caplog.at_level(logging.INFO): + result = CliRunner().invoke( + cli, + [ + *OFFLINE, + "check-iocs", + "--load-module", + str(module_path), + str(results_folder), + ], + ) + + assert result.exit_code == 0, result.output + assert "loaded module checked 1 results" in caplog.text diff --git a/tests/common/test_cmd_plugins.py b/tests/common/test_cmd_plugins.py new file mode 100644 index 0000000..adf9322 --- /dev/null +++ b/tests/common/test_cmd_plugins.py @@ -0,0 +1,236 @@ +import json +from types import SimpleNamespace + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.common.cmd_plugins import plugins +from mvt.common.module import MVTModule +from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP +from mvt.common.updates import PluginUpdates +from mvt.ios.cli import cli as ios_cli + + +class ExampleModule(MVTModule): + pass + + +class AnotherModule(MVTModule): + pass + + +class FakeDistribution: + def __init__(self, name, version="1.0.0", direct_url=None): + self.name = name + self.version = version + self.direct_url = direct_url + + def read_text(self, file_name): + if file_name == "direct_url.json" and self.direct_url is not None: + return json.dumps(self.direct_url) + return None + + +def _entry_point(name, distribution, modules=None, exception=None): + def load(): + if exception is not None: + raise exception + return modules + + return SimpleNamespace( + name=name, value="example_plugin:modules", dist=distribution, load=load + ) + + +def _run(command, arguments): + # Keep rich from wrapping the table while its content is being asserted. + return CliRunner().invoke(command, arguments, env={"COLUMNS": "200"}) + + +def _table_rows(output): + """Return the content of the table rows, without the header and the box.""" + return [ + [cell.strip() for cell in line.strip().strip("│").split("│")] + for line in output.splitlines() + if "│" in line + ] + + +def _table_header(output): + for line in output.splitlines(): + if "┃" in line: + return [cell.strip() for cell in line.strip().strip("┃").split("┃")] + return [] + + +def _install(monkeypatch, distributions, entry_points): + monkeypatch.setattr( + "mvt.common.cmd_plugins.installed_plugin_distributions", + lambda: distributions, + ) + monkeypatch.setattr( + "mvt.common.cmd_plugins.importlib.metadata.entry_points", + lambda *, group: entry_points.get(group, []), + ) + + +def test_plugins_is_a_builtin_command_of_the_mvt_cli_only(): + assert mvt_cli.commands["plugins"] is plugins + assert "plugins" not in ios_cli.commands + assert "plugins" not in android_cli.commands + + +def test_list_shows_what_every_plugin_contributes(monkeypatch): + index_plugin = FakeDistribution("example-plugin", version="1.2.0") + repository_plugin = FakeDistribution( + "repository-plugin", + version="0.1.0", + direct_url={ + "url": "https://example.org/plugin.git", + "vcs_info": {"vcs": "git", "commit_id": "b" * 40}, + }, + ) + local_plugin = FakeDistribution( + "local-plugin", + direct_url={"url": "file:///plugins", "dir_info": {"editable": True}}, + ) + _install( + monkeypatch, + [index_plugin, local_plugin, repository_plugin], + { + MODULES_ENTRY_POINT_GROUP: [ + _entry_point( + "example", index_plugin, modules=[ExampleModule, AnotherModule] + ), + _entry_point("local", local_plugin, modules=lambda: [ExampleModule]), + ], + IOS_CLI_PLUGIN_GROUP: [_entry_point("summarize", repository_plugin)], + ANDROID_CLI_PLUGIN_GROUP: [_entry_point("triage", local_plugin)], + NEUTRAL_CLI_PLUGIN_GROUP: [_entry_point("report", repository_plugin)], + }, + ) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + # Plugins are listed by name, with the modules and the commands each of + # them contributes. + assert _table_header(result.output) == [ + "Name", + "Version", + "Origin", + "Modules", + "Commands", + ] + assert _table_rows(result.output) == [ + ["example-plugin", "1.2.0", "pypi", "2", "-"], + ["local-plugin", "1.0.0", "local", "1", "triage"], + ["repository-plugin", "0.1.0", "git+bbbbbbbb", "0", "report, summarize"], + ] + + +def test_list_reports_a_broken_module_entry_point(monkeypatch): + plugin = FakeDistribution("broken-plugin") + _install( + monkeypatch, + [plugin], + { + MODULES_ENTRY_POINT_GROUP: [ + _entry_point( + "broken", plugin, exception=ImportError("missing dependency") + ) + ] + }, + ) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + assert _table_rows(result.output) == [ + ["broken-plugin", "1.0.0", "pypi", "error", "-"] + ] + + +def test_list_without_plugins(monkeypatch): + _install(monkeypatch, [], {}) + + result = _run(plugins, ["list"]) + + assert result.exit_code == 0 + assert result.output == "No MVT plugins are installed.\n" + + +def test_check_updates_prints_the_findings_and_ignores_the_throttle(monkeypatch): + findings = [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ] + _install(monkeypatch, [FakeDistribution("example-plugin")], {}) + monkeypatch.setattr(PluginUpdates, "check", lambda self: findings) + monkeypatch.setattr( + PluginUpdates, + "should_check", + lambda self: pytest.fail("an explicit check must not be throttled"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "Plugin updates available:" in result.output + assert "example-plugin 1.0.0 → 1.2.0" in result.output + assert "Upgrade with: pip install -U example-plugin" in result.output + assert "MVT does not install plugin updates." in result.output + + +def test_check_updates_without_available_updates(monkeypatch): + _install(monkeypatch, [FakeDistribution("example-plugin")], {}) + monkeypatch.setattr(PluginUpdates, "check", lambda self: []) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "All plugins are up to date." in result.output + + +def test_check_updates_without_plugins(monkeypatch): + _install(monkeypatch, [], {}) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("nothing must be checked without plugins"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "No MVT plugins are installed." in result.output + + +def test_check_updates_without_network_access(monkeypatch): + monkeypatch.setattr("mvt.common.cmd_plugins.settings.NETWORK_ACCESS_ALLOWED", False) + monkeypatch.setattr( + "mvt.common.cmd_plugins.installed_plugin_distributions", + lambda: pytest.fail("plugins must not be listed without network access"), + ) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("nothing must be checked without network access"), + ) + + result = _run(plugins, ["check-updates"]) + + assert result.exit_code == 0 + assert "Network access is disabled" in result.output diff --git a/tests/common/test_command.py b/tests/common/test_command.py index 4dbfe1a..8e09f57 100644 --- a/tests/common/test_command.py +++ b/tests/common/test_command.py @@ -68,6 +68,123 @@ class CustomDependsOnBuiltin(RecordingModule): dependencies = (FirstModule,) +class ReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + def run(self): + super().run() + self.results = ["replacement"] + + +class OtherReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + +class UnrelatedReplacementModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = IndependentModule + + +class ReplacementOfReplacementModule(ReplacementModule): + supported_commands = (("ios", "check-backup"),) + replaces = ReplacementModule + + def run(self): + super().run() + self.results = ["replacement of replacement"] + + +class ReplacesOwnDependencyModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + dependencies = (FirstModule,) + + +class DisabledReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + enabled = False + + +class MutualReplacementOne(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class MutualReplacementTwo(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = MutualReplacementOne + + +MutualReplacementOne.replaces = MutualReplacementTwo + + +class CycleOneModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class CycleTwoModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = CycleOneModule + + +class CycleThreeModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + replaces = CycleTwoModule + + +CycleOneModule.replaces = CycleThreeModule + + +class UnavailableDependencyModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + + +class ReplacementMissingDependency(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + dependencies = (UnavailableDependencyModule,) + + +class SharedSlugModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + slug = "shared_slug" + + +class OtherSharedSlugModule(RecordingModule): + supported_commands = (("ios", "check-backup"),) + slug = "shared_slug" + + +class ReplacementKeepingTheSlug(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + slug = "first_module" + + +class SameNameReplacementModule(FirstModule): + supported_commands = (("ios", "check-backup"),) + replaces = FirstModule + + def run(self): + super().run() + self.results = ["replacement"] + + +# Modules replacing a built-in one usually keep its class name, which is the +# name `--module` matches on. +SameNameReplacementModule.__name__ = "FirstModule" + + +def logged_substitutions(caplog) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.INFO and "replaces module" in record.getMessage() + ] + + class RecordingCommand(Command): def init(self): self.initialized = True @@ -157,7 +274,7 @@ class TestCommand: assert not hasattr(cmd, "initialized") assert "Circular module dependency detected" in caplog.text - def test_unavailable_dependency_warns_and_stops(self, caplog): + def test_unavailable_dependency_only_skips_the_dependent_module(self, caplog): class UnavailableModule(RecordingModule): pass @@ -165,14 +282,97 @@ class TestCommand: dependencies = (UnavailableModule,) cmd = RecordingCommand() - cmd.modules = [DependentModule] + cmd.modules = [DependentModule, IndependentModule, FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "FirstModule"] + assert cmd.initialized + assert "Module DependentModule will be SKIPPED" in caplog.text + assert "depends on module UnavailableModule" in caplog.text + + def test_modules_depending_on_a_skipped_module_are_skipped_too(self, caplog): + class UnavailableModule(RecordingModule): + pass + + class SkippedModule(RecordingModule): + dependencies = (UnavailableModule,) + + class DependsOnSkippedModule(RecordingModule): + dependencies = (SkippedModule,) + + class DependsOnTheChain(RecordingModule): + dependencies = (DependsOnSkippedModule,) + + cmd = RecordingCommand() + cmd.modules = [ + DependsOnTheChain, + DependsOnSkippedModule, + SkippedModule, + IndependentModule, + ] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule"] + skip_warnings = [ + record.getMessage() + for record in caplog.records + if "will be SKIPPED" in record.getMessage() + ] + assert len(skip_warnings) == 3 + assert [warning.split()[1] for warning in skip_warnings] == [ + "SkippedModule", + "DependsOnSkippedModule", + "DependsOnTheChain", + ] + # Every warning names the root cause: the module missing a dependency + # and the dependency it is missing. + assert all("UnavailableModule" in warning for warning in skip_warnings) + assert all("module SkippedModule" in warning for warning in skip_warnings[1:]) + + def test_explicitly_selected_module_with_missing_dependency_runs_nothing( + self, caplog + ): + class UnavailableModule(RecordingModule): + pass + + class DependentModule(RecordingModule): + dependencies = (UnavailableModule,) + + cmd = RecordingCommand(module_name="DependentModule") + cmd.modules = [DependentModule, IndependentModule] with caplog.at_level(logging.WARNING): cmd.run() assert RecordingModule.run_order == [] - assert not hasattr(cmd, "initialized") - assert "depends on unavailable module UnavailableModule" in caplog.text + assert "Module DependentModule will be SKIPPED" in caplog.text + assert "No modules will be run" in caplog.text + # Nothing else was selected, so the warning must not promise that the + # analysis continues right before saying that it does not. + assert "The rest of the analysis will still run" not in caplog.text + + def test_unaffected_dependency_chains_keep_their_order(self, caplog): + class UnavailableModule(RecordingModule): + pass + + class SkippedModule(RecordingModule): + dependencies = (UnavailableModule,) + + cmd = RecordingCommand() + cmd.modules = [ThirdModule, SkippedModule, SecondModule, FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "FirstModule", + "SecondModule", + "ThirdModule", + ] def test_custom_modules_are_filtered_before_ordering(self): cmd = RecordingCommand() @@ -220,3 +420,376 @@ class TestCommand: cmd.run() assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"] + + def test_custom_module_replaces_builtin(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"] + assert cmd.module_replacements == {FirstModule: ReplacementModule} + assert ( + "Module ReplacementModule from" in caplog.text + and "replaces module FirstModule" in caplog.text + ) + + def test_modules_without_replaces_all_run(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [CustomIOSBackupModule] + + cmd.run() + + assert RecordingModule.run_order == ["FirstModule", "CustomIOSBackupModule"] + assert cmd.module_replacements == {} + + def test_unavailable_replaced_module_is_ignored(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"] + assert cmd.module_replacements == {} + assert "replaces module FirstModule" not in caplog.text + + def test_dependencies_are_resolved_to_the_replacement(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [ReplacementModule] + + cmd.run() + + assert RecordingModule.run_order == ["ReplacementModule", "SecondModule"] + second = next( + module for module in cmd.executed if isinstance(module, SecondModule) + ) + assert isinstance(second.dependency_modules[FirstModule], ReplacementModule) + assert second.results == ["replacement", "second"] + + def test_replacement_with_unavailable_dependency_skips_its_dependents( + self, caplog + ): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, SecondModule, IndependentModule] + cmd.custom_modules = [ReplacementMissingDependency] + + with caplog.at_level(logging.INFO): + cmd.run() + + # The replacement cannot run, and the module it replaced was dropped + # from the run by the replacement, so neither of them produces + # results and the module depending on the replaced one is skipped. + assert RecordingModule.run_order == ["IndependentModule"] + assert ( + "Module ReplacementMissingDependency will be SKIPPED: it depends " + "on module UnavailableDependencyModule, which is not available in " + "this command." in caplog.text + ) + # The skipped dependent is told which module it actually depends on, + # and the module class its author declared. + assert ( + "Module SecondModule will be SKIPPED: it depends on module " + "ReplacementMissingDependency (replacing module FirstModule), " + "itself skipped for depending on unavailable module " + "UnavailableDependencyModule." in caplog.text + ) + + def test_selected_replacement_with_unavailable_dependency_runs_nothing( + self, caplog, tmp_path + ): + cmd = RecordingCommand(module_name="FirstModule", results_path=str(tmp_path)) + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementMissingDependency] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert ( + "Module FirstModule was replaced by module " + "ReplacementMissingDependency, which is run in its place." + in caplog.text + ) + assert "Module ReplacementMissingDependency will be SKIPPED" in caplog.text + assert "No modules will be run" in caplog.text + # Nothing else was selected, so the warnings must not promise that the + # analysis continues right before saying that it does not. + assert "The rest of the analysis will still run" not in caplog.text + # No module ran, so no results were stored next to the command log. + assert [path.name for path in tmp_path.iterdir()] == ["command.log"] + + def test_chained_replacements_are_resolved(self): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [SecondModule, FirstModule] + cmd.custom_modules = [ReplacementModule, ReplacementOfReplacementModule] + + cmd.run() + + assert RecordingModule.run_order == [ + "ReplacementOfReplacementModule", + "SecondModule", + ] + second = next( + module for module in cmd.executed if isinstance(module, SecondModule) + ) + assert second.results == ["replacement of replacement", "second"] + + def test_replacement_which_is_not_a_subclass_warns(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [IndependentModule] + cmd.custom_modules = [UnrelatedReplacementModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["UnrelatedReplacementModule"] + assert "is not a subclass of it" in caplog.text + + def test_multiple_modules_replacing_the_same_module_warn(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementModule, OtherReplacementModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "ReplacementModule", + "OtherReplacementModule", + ] + assert cmd.module_replacements == {FirstModule: ReplacementModule} + assert ( + "Modules ReplacementModule and OtherReplacementModule both replace " + "module FirstModule. Both of them will run, FirstModule will not, " + "and modules depending on FirstModule will use the results of " + "ReplacementModule." in caplog.text + ) + assert "overwrite each other's results file" in caplog.text + + def test_module_replacing_its_own_dependency_runs(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacesOwnDependencyModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == ["ReplacesOwnDependencyModule"] + assert ( + "Module ReplacesOwnDependencyModule depends on module FirstModule, " + "which it also replaces" in caplog.text + ) + + def test_literal_self_dependency_is_still_circular(self, caplog): + class SelfDependentModule(RecordingModule): + pass + + SelfDependentModule.dependencies = (SelfDependentModule,) + + cmd = RecordingCommand() + cmd.modules = [SelfDependentModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert "Circular module dependency detected" in caplog.text + + def test_disabled_replacement_keeps_the_replaced_module(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [DisabledReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["FirstModule"] + assert cmd.module_replacements == {} + assert logged_substitutions(caplog) == [] + + def test_modules_replacing_each_other_all_run(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [MutualReplacementOne, MutualReplacementTwo] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [ + "MutualReplacementOne", + "MutualReplacementTwo", + ] + assert cmd.module_replacements == {} + assert ( + "Modules MutualReplacementOne, MutualReplacementTwo replace each " + "other in a cycle" in caplog.text + ) + assert logged_substitutions(caplog) == [] + + def test_replacement_cycle_of_three_modules_all_run(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [CycleOneModule, CycleTwoModule, CycleThreeModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == [ + "CycleOneModule", + "CycleTwoModule", + "CycleThreeModule", + ] + assert cmd.module_replacements == {} + assert ( + "Modules CycleOneModule, CycleThreeModule, CycleTwoModule replace " + "each other in a cycle" in caplog.text + ) + assert logged_substitutions(caplog) == [] + + def test_selected_replaced_module_name_runs_the_replacement(self, caplog): + cmd = RecordingCommand(module_name="FirstModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.run() + + assert RecordingModule.run_order == ["ReplacementModule"] + assert ( + "Module FirstModule was replaced by module ReplacementModule" + in caplog.text + ) + + def test_unknown_selected_module_warns_and_stops(self, caplog): + cmd = RecordingCommand(module_name="NoSuchModule") + cmd.name = "check-backup" + cmd.modules = [FirstModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [] + assert not hasattr(cmd, "initialized") + assert ( + "No module named NoSuchModule is available for the check-backup " + "command" in caplog.text + ) + + def test_selected_module_name_matches_the_replacement(self): + cmd = RecordingCommand(module_name="FirstModule") + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [SameNameReplacementModule] + + cmd.run() + + assert len(cmd.executed) == 1 + assert isinstance(cmd.executed[0], SameNameReplacementModule) + assert cmd.executed[0].results == ["replacement"] + + def test_list_modules_reflects_replacements(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [ReplacementModule] + + with caplog.at_level(logging.INFO): + cmd.list_modules() + + listed = [ + record.getMessage() + for record in caplog.records + if "Modules from" in record.getMessage() + ] + assert any("ReplacementModule" in message for message in listed) + assert not any("FirstModule" in message for message in listed) + + def test_modules_sharing_a_slug_are_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.custom_modules = [SharedSlugModule, OtherSharedSlugModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert RecordingModule.run_order == [ + "SharedSlugModule", + "OtherSharedSlugModule", + ] + collisions = [ + record.getMessage() + for record in caplog.records + if "both use the slug" in record.getMessage() + ] + assert len(collisions) == 1 + assert "Modules SharedSlugModule from" in collisions[0] + assert "and OtherSharedSlugModule from" in collisions[0] + assert "both use the slug shared_slug" in collisions[0] + assert "overwrites the results of the other in shared_slug.json" in ( + collisions[0] + ) + + def test_replacement_keeping_the_replaced_slug_is_not_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule] + cmd.custom_modules = [ReplacementKeepingTheSlug] + + with caplog.at_level(logging.WARNING): + cmd.run() + + # The replaced module is no longer part of the run, so taking over its + # slug is what the replacement is for, not a collision. + assert RecordingModule.run_order == ["ReplacementKeepingTheSlug"] + assert ReplacementKeepingTheSlug.get_slug() == FirstModule.get_slug() + assert "both use the slug" not in caplog.text + + def test_modules_with_distinct_slugs_are_not_reported(self, caplog): + cmd = RecordingCommand() + cmd.platform = "ios" + cmd.name = "check-backup" + cmd.modules = [FirstModule, IndependentModule] + cmd.custom_modules = [CustomIOSBackupModule] + + with caplog.at_level(logging.WARNING): + cmd.run() + + assert "both use the slug" not in caplog.text diff --git a/tests/common/test_command_modules.py b/tests/common/test_command_modules.py new file mode 100644 index 0000000..8f79ce9 --- /dev/null +++ b/tests/common/test_command_modules.py @@ -0,0 +1,27 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES +from mvt.android.modules.androidqf import ANDROIDQF_MODULES +from mvt.android.modules.backup import BACKUP_MODULES as ANDROID_BACKUP_MODULES +from mvt.android.modules.bugreport import BUGREPORT_MODULES +from mvt.android.modules.intrusion_logs import INTRUSION_LOGS_MODULES +from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES +from mvt.ios.modules.backup import BACKUP_MODULES as IOS_BACKUP_MODULES +from mvt.ios.modules.fs import FS_MODULES +from mvt.ios.modules.mixed import MIXED_MODULES + + +def test_the_check_iocs_lists_are_the_families_of_their_platform(): + # The CLI reads these same lists, so nothing composing one elsewhere can + # drift from what the command runs. This pins what the lists are composed + # of. + assert IOS_CHECK_IOCS_MODULES == IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES + assert ANDROID_CHECK_IOCS_MODULES == ( + ANDROID_BACKUP_MODULES + + BUGREPORT_MODULES + + ANDROIDQF_MODULES + + INTRUSION_LOGS_MODULES + ) diff --git a/tests/common/test_module_loader.py b/tests/common/test_module_loader.py index 15b6052..527d01c 100644 --- a/tests/common/test_module_loader.py +++ b/tests/common/test_module_loader.py @@ -1,12 +1,19 @@ +from pathlib import Path + import pytest +from mvt.common.cli_plugins import _module_name_for_path as _command_module_name from mvt.common.module import MVTModule from mvt.common.module_loader import ( CustomModuleLoadError, + _module_name_for_path, + get_module_logger, + get_plugin_logger, load_custom_modules, load_custom_modules_from_path, module_supports_command, ) +from mvt.ios.modules.mixed.whatsapp import Whatsapp MODULE_TEMPLATE = """ @@ -144,3 +151,66 @@ def test_module_supports_command_honors_supported_commands(tmp_path): assert module_supports_command(module, "ios", "check-backup") assert not module_supports_command(module, "ios", "check-fs") + + +def test_get_module_logger_keeps_builtin_names(): + assert get_module_logger(Whatsapp).name == "mvt.ios.modules.mixed.whatsapp" + + +def test_get_module_logger_parents_package_modules_under_mvt_ext(): + class PackageModule(MVTModule): + pass + + PackageModule.__module__ = "some_plugin_package.ios.custom" + + assert ( + get_module_logger(PackageModule).name + == "mvt.ext.some_plugin_package.ios.custom" + ) + + +def test_get_module_logger_strips_the_plugin_package_prefix(): + class PluginModule(MVTModule): + pass + + PluginModule.__module__ = "mvt_plugin_example_org.ios.custom" + + assert get_module_logger(PluginModule).name == "mvt.ext.example_org.ios.custom" + + +def test_get_module_logger_only_strips_the_prefix_from_the_top_level(): + class NestedModule(MVTModule): + pass + + NestedModule.__module__ = "other_package.mvt_plugin_sub" + + assert get_module_logger(NestedModule).name == "mvt.ext.other_package.mvt_plugin_sub" + + +def test_get_module_logger_names_path_modules_after_their_file(tmp_path): + module_path = _write_module(tmp_path / "my_custom_module.py", "PathModule") + module = load_custom_modules_from_path(str(module_path))[0] + + assert get_module_logger(module).name == "mvt.ext.my_custom_module" + + +def test_get_plugin_logger_uses_the_same_namespace_as_modules(): + assert ( + get_plugin_logger("mvt_plugin_example_org.commands.summarize").name + == "mvt.ext.example_org.commands.summarize" + ) + assert get_plugin_logger("example_plugin.cli").name == "mvt.ext.example_plugin.cli" + + +def test_get_plugin_logger_keeps_builtin_names(): + assert get_plugin_logger("mvt.ios.cli").name == "mvt.ios.cli" + + +def test_get_plugin_logger_names_loaded_files_after_the_file(): + # A file loaded with --load-command or --load-module is imported under a + # mangled name. The log names the file instead. + command_name = _command_module_name(Path("/tmp/case_summary.py")) + module_name = _module_name_for_path(Path("/tmp/my_custom_module.py")) + + assert get_plugin_logger(command_name).name == "mvt.ext.case_summary" + assert get_plugin_logger(module_name).name == "mvt.ext.my_custom_module" diff --git a/tests/common/test_plugin.py b/tests/common/test_plugin.py new file mode 100644 index 0000000..255f2e8 --- /dev/null +++ b/tests/common/test_plugin.py @@ -0,0 +1,34 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import mvt.plugin +from mvt.android.modules.backup.base import BackupModule +from mvt.common.config import settings + +from ..plugin_fixtures import run_isolated_python + + +def test_the_exported_names_are_the_public_names(): + public = {name for name in vars(mvt.plugin) if not name.startswith("_")} + + assert public == set(mvt.plugin.__all__) + assert mvt.plugin.settings is settings + assert mvt.plugin.AndroidBackupModule is BackupModule + + +def test_the_surface_imports_before_anything_else_of_mvt(tmp_path): + # A plugin can import the surface as its first import of MVT. The + # subprocess gets a temporary home because importing MVT writes its + # configuration file. + result = run_isolated_python( + "from mvt.plugin import IOSExtraction, MVT_VERSION, settings\n" + "assert MVT_VERSION\n" + "assert settings.NETWORK_TIMEOUT > 0\n" + "assert IOSExtraction.__name__ == 'IOSExtraction'\n", + home=tmp_path, + ) + + assert result.returncode == 0, result.stderr + assert result.stderr == "" diff --git a/tests/common/test_plugin_config.py b/tests/common/test_plugin_config.py new file mode 100644 index 0000000..9bcf9eb --- /dev/null +++ b/tests/common/test_plugin_config.py @@ -0,0 +1,391 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import stat +import sys +from typing import Optional + +import pytest +import yaml + +from mvt.common.plugin_config import ( + MVTPluginSettings, + PluginConfigLoadError, + plugin_config_folder, + plugin_config_path, + plugin_data_folder, + plugin_env_prefix, +) + + +class ExamplePluginSettings(MVTPluginSettings): + plugin_name = "example-plugin" + + API_KEY: Optional[str] = None + CACHE_FOLDER: str = "cache" + MAX_RESULTS: int = 25 + + +class OtherPluginSettings(MVTPluginSettings): + plugin_name = "other-plugin" + + API_KEY: Optional[str] = None + + +@pytest.fixture +def config_folder(tmp_path, monkeypatch): + monkeypatch.setattr( + "mvt.common.plugin_config.user_config_dir", + lambda *args, **kwargs: str(tmp_path), + ) + return tmp_path + + +@pytest.fixture +def data_folder(tmp_path, monkeypatch): + folder = tmp_path / "data" + monkeypatch.setattr( + "mvt.common.plugin_config.user_data_dir", + lambda *args, **kwargs: str(folder), + ) + return folder + + +def _write_plugin_file(plugin_name, values): + config_path = plugin_config_path(plugin_name) + os.makedirs(os.path.dirname(config_path), exist_ok=True) + content = values if isinstance(values, str) else yaml.dump(values) + with open(config_path, "w") as config_file: + config_file.write(content) + return config_path + + +def test_plugin_paths_and_prefixes_are_namespaced(config_folder): + assert plugin_config_folder() == str(config_folder / "plugins") + assert plugin_config_path("example-plugin") == str( + config_folder / "plugins" / "example-plugin.yaml" + ) + assert plugin_env_prefix("example-plugin") == "MVT_PLUGIN_EXAMPLE_PLUGIN_" + assert plugin_env_prefix("other-plugin") == "MVT_PLUGIN_OTHER_PLUGIN_" + + +def test_defaults_are_used_without_file_or_environment(config_folder): + settings = ExamplePluginSettings.load() + + assert settings.API_KEY is None + assert settings.CACHE_FOLDER == "cache" + assert settings.MAX_RESULTS == 25 + assert not os.path.exists(plugin_config_path("example-plugin")) + + +def test_values_are_loaded_from_the_plugin_file(config_folder): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-file" + assert settings.MAX_RESULTS == 5 + assert settings.CACHE_FOLDER == "cache" + + +def test_environment_overrides_the_plugin_file(config_folder, monkeypatch): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment") + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-environment" + assert settings.MAX_RESULTS == 5 + + +def test_arguments_override_the_environment_and_the_plugin_file( + config_folder, monkeypatch +): + _write_plugin_file("example-plugin", {"API_KEY": "from-file", "MAX_RESULTS": 5}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "from-environment") + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "10") + + settings = ExamplePluginSettings(API_KEY="from-argument") + + assert settings.API_KEY == "from-argument" + assert settings.MAX_RESULTS == 10 + + +def test_save_and_load_round_trip(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + settings.MAX_RESULTS = 100 + + settings.save() + + config_path = plugin_config_path("example-plugin") + assert os.path.isfile(config_path) + with open(config_path) as config_file: + assert yaml.safe_load(config_file) == { + "API_KEY": "saved-key", + "MAX_RESULTS": 100, + } + + reloaded = ExamplePluginSettings.load() + assert reloaded.API_KEY == "saved-key" + assert reloaded.MAX_RESULTS == 100 + assert reloaded.CACHE_FOLDER == "cache" + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_saved_file_is_only_readable_by_the_user(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + + settings.save() + + config_path = plugin_config_path("example-plugin") + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 + folder_mode = stat.S_IMODE(os.stat(plugin_config_folder()).st_mode) + assert folder_mode & 0o077 == 0 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_save_restricts_the_permissions_of_an_existing_file(config_folder): + config_path = _write_plugin_file("example-plugin", {"API_KEY": "from-file"}) + os.chmod(config_path, 0o644) + + settings = ExamplePluginSettings.load() + settings.MAX_RESULTS = 100 + settings.save() + + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 + assert os.listdir(plugin_config_folder()) == ["example-plugin.yaml"] + + +def test_save_only_persists_non_default_values(config_folder): + settings = ExamplePluginSettings.load() + settings.CACHE_FOLDER = "another-cache" + + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"CACHE_FOLDER": "another-cache"} + + +def test_save_does_not_persist_values_coming_from_the_environment( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + + settings = ExamplePluginSettings.load() + assert settings.API_KEY == "environment-secret" + settings.MAX_RESULTS = 100 + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"MAX_RESULTS": 100} + + +def test_an_invalid_environment_variable_still_protects_the_other_values( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS", "not-an-int") + + settings = ExamplePluginSettings(MAX_RESULTS=100) + assert settings.API_KEY == "environment-secret" + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + saved_values = yaml.safe_load(config_file) + assert saved_values == {"MAX_RESULTS": 100} + assert "API_KEY" not in saved_values + + +def test_save_persists_values_which_differ_from_the_environment( + config_folder, monkeypatch +): + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "environment-secret") + + settings = ExamplePluginSettings.load() + settings.API_KEY = "chosen-key" + settings.save() + + with open(plugin_config_path("example-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"API_KEY": "chosen-key"} + + +def test_save_does_not_write_the_mvt_configuration_file(config_folder): + settings = ExamplePluginSettings.load() + settings.API_KEY = "saved-key" + + settings.save() + + assert os.listdir(config_folder) == ["plugins"] + + +def test_unknown_keys_in_the_plugin_file_are_ignored(config_folder): + _write_plugin_file( + "example-plugin", + {"API_KEY": "from-file", "UNKNOWN_SETTING": "ignored"}, + ) + + settings = ExamplePluginSettings.load() + + assert settings.API_KEY == "from-file" + assert not hasattr(settings, "UNKNOWN_SETTING") + + +def test_unparsable_plugin_file_is_reported_with_its_path(config_folder): + config_path = _write_plugin_file("example-plugin", "API_KEY: [unclosed\n") + + with pytest.raises(PluginConfigLoadError) as raised: + ExamplePluginSettings.load() + + assert config_path in str(raised.value) + + +def test_plugin_file_which_is_not_a_mapping_is_reported_with_its_path(config_folder): + config_path = _write_plugin_file("example-plugin", "- one\n- two\n") + + with pytest.raises(PluginConfigLoadError) as raised: + ExamplePluginSettings.load() + + assert config_path in str(raised.value) + assert "mapping of setting names" in str(raised.value) + + +def test_plugins_do_not_interfere_with_each_other(config_folder, monkeypatch): + _write_plugin_file("other-plugin", {"API_KEY": "other-file-key"}) + monkeypatch.setenv("MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY", "example-environment-key") + + example_settings = ExamplePluginSettings.load() + other_settings = OtherPluginSettings.load() + + assert example_settings.API_KEY == "example-environment-key" + assert other_settings.API_KEY == "other-file-key" + + example_settings.MAX_RESULTS = 100 + example_settings.save() + assert sorted(os.listdir(plugin_config_folder())) == [ + "example-plugin.yaml", + "other-plugin.yaml", + ] + with open(plugin_config_path("other-plugin")) as config_file: + assert yaml.safe_load(config_file) == {"API_KEY": "other-file-key"} + + +def test_subclass_without_plugin_name_is_rejected(): + with pytest.raises(TypeError, match="plugin_name"): + + class MissingNameSettings(MVTPluginSettings): + API_KEY: Optional[str] = None + + +def test_subclass_with_invalid_plugin_name_is_rejected(): + with pytest.raises(ValueError, match="Invalid plugin name"): + + class InvalidNameSettings(MVTPluginSettings): + plugin_name = "Bad/Name" + + +def test_underscores_are_not_allowed_in_plugin_names(): + # Underscores are replaced by dashes in the environment prefix, so allowing + # both would let two plugin names share one environment namespace. + with pytest.raises(ValueError, match="Invalid plugin name"): + + class UnderscoreNameSettings(MVTPluginSettings): + plugin_name = "under_score" + + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_config_path("under_score") + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_env_prefix("under_score") + + +@pytest.mark.parametrize( + "plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""] +) +def test_unsafe_plugin_names_have_no_configuration_path(plugin_name): + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_config_path(plugin_name) + + +def test_data_folder_is_namespaced_and_created(data_folder): + folder = plugin_data_folder("example-plugin") + + assert folder == str(data_folder / "plugin-data" / "example-plugin") + assert os.path.isdir(folder) + + +def test_data_folder_can_be_requested_repeatedly(data_folder): + folder = plugin_data_folder("example-plugin") + with open(os.path.join(folder, "kept.json"), "w") as data_file: + data_file.write("{}") + + assert plugin_data_folder("example-plugin") == folder + assert os.listdir(folder) == ["kept.json"] + + +@pytest.mark.skipif( + sys.platform == "win32", reason="POSIX file permissions are not available" +) +def test_data_folder_is_only_accessible_by_the_user(data_folder): + folder = plugin_data_folder("example-plugin") + + assert stat.S_IMODE(os.stat(folder).st_mode) & 0o077 == 0 + parent_mode = stat.S_IMODE(os.stat(os.path.dirname(folder)).st_mode) + assert parent_mode & 0o077 == 0 + + +def test_plugins_get_their_own_data_folder(data_folder): + example_folder = plugin_data_folder("example-plugin") + other_folder = plugin_data_folder("other-plugin") + + assert example_folder != other_folder + assert sorted(os.listdir(data_folder / "plugin-data")) == [ + "example-plugin", + "other-plugin", + ] + + +def test_data_folder_does_not_touch_the_configuration_folder( + config_folder, data_folder +): + plugin_data_folder("example-plugin") + + assert not os.path.exists(plugin_config_folder()) + + +@pytest.mark.parametrize( + "plugin_name", ["../escape", "folder/name", "UPPER", "-dash", ""] +) +def test_unsafe_plugin_names_have_no_data_folder(data_folder, plugin_name): + with pytest.raises(ValueError, match="Invalid plugin name"): + plugin_data_folder(plugin_name) + + assert not os.path.exists(data_folder) + + +def test_settings_class_knows_its_data_folder(data_folder): + folder = ExamplePluginSettings.data_folder() + + assert folder == plugin_data_folder("example-plugin") + assert os.path.isdir(folder) + assert OtherPluginSettings.data_folder() != folder + + +def test_settings_instance_uses_the_same_data_folder(config_folder, data_folder): + settings = ExamplePluginSettings.load() + + assert settings.data_folder() == ExamplePluginSettings.data_folder() + + +def test_subclass_without_its_own_name_shares_the_data_folder(data_folder): + class InheritingSettings(ExamplePluginSettings): + pass + + assert InheritingSettings.data_folder() == ExamplePluginSettings.data_folder() diff --git a/tests/common/test_plugin_updates.py b/tests/common/test_plugin_updates.py new file mode 100644 index 0000000..9c45cea --- /dev/null +++ b/tests/common/test_plugin_updates.py @@ -0,0 +1,711 @@ +import json +import shlex +from datetime import datetime, timedelta +from types import SimpleNamespace + +import pytest + +from mvt.common import logo +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP +from mvt.common.updates import ( + MVTUpdates, + PluginUpdates, + installed_plugin_distributions, +) + + +REPOSITORY_URL = "https://example.org/plugin.git" +INSTALLED_COMMIT = "a" * 40 +REMOTE_COMMIT = "b" * 40 + + +class FakeDistribution: + def __init__(self, name, version="1.0.0", direct_url=None): + self.name = name + self.version = version + self.direct_url = direct_url + + def read_text(self, file_name): + if file_name == "direct_url.json" and self.direct_url is not None: + return json.dumps(self.direct_url) + return None + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self.payload = payload or {} + + def json(self): + return self.payload + + +def _entry_point(name, distribution, value="plugin:modules"): + return SimpleNamespace(name=name, value=value, dist=distribution) + + +def _git_distribution(requested_revision=None, commit=INSTALLED_COMMIT): + vcs_info = {"vcs": "git", "commit_id": commit} + if requested_revision: + vcs_info["requested_revision"] = requested_revision + + return FakeDistribution( + "example-plugin", + direct_url={"url": REPOSITORY_URL, "vcs_info": vcs_info}, + ) + + +def _fake_git(stdout="", returncode=0, exception=None, calls=None): + def run(command, **kwargs): + if calls is not None: + calls.append((command, kwargs)) + if exception is not None: + raise exception + return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="") + + return run + + +@pytest.fixture +def data_folder(tmp_path, monkeypatch): + folder = tmp_path / "mvt-data" + monkeypatch.setattr("mvt.common.updates.MVT_DATA_FOLDER", str(folder)) + return folder + + +@pytest.fixture +def one_plugin(monkeypatch): + def install(distribution): + monkeypatch.setattr( + "mvt.common.updates.installed_plugin_distributions", + lambda: [distribution], + ) + return distribution + + return install + + +def test_installed_plugin_distributions_covers_every_plugin_group(monkeypatch): + zeta = FakeDistribution("zeta-plugin") + alpha = FakeDistribution("alpha-plugin") + neutral = FakeDistribution("neutral-plugin") + + def entry_points(*, group): + if group == MODULES_ENTRY_POINT_GROUP: + return [_entry_point("zeta", zeta), _entry_point("alpha", alpha)] + if group == IOS_CLI_PLUGIN_GROUP: + return [_entry_point("zeta-ios", zeta)] + if group == ANDROID_CLI_PLUGIN_GROUP: + return [_entry_point("alpha-android", alpha)] + if group == NEUTRAL_CLI_PLUGIN_GROUP: + return [_entry_point("shared", neutral)] + return [] + + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", entry_points + ) + + distributions = installed_plugin_distributions() + + assert [distribution.name for distribution in distributions] == [ + "alpha-plugin", + "neutral-plugin", + "zeta-plugin", + ] + + +def test_installed_plugin_distributions_skips_mvt_and_orphan_entry_points(monkeypatch): + entry_points = [ + _entry_point("builtin", FakeDistribution("mvt")), + SimpleNamespace(name="orphan", value="plugin:modules", dist=None), + _entry_point("plugin", FakeDistribution("example-plugin")), + ] + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", + lambda **kwargs: entry_points, + ) + + distributions = installed_plugin_distributions() + + assert [distribution.name for distribution in distributions] == ["example-plugin"] + + +def test_installed_plugin_distributions_survives_broken_metadata(monkeypatch, caplog): + def entry_points(*, group): + raise RuntimeError("invalid package metadata") + + monkeypatch.setattr( + "mvt.common.updates.importlib.metadata.entry_points", entry_points + ) + + assert installed_plugin_distributions() == [] + assert "Unable to discover installed plugin packages" in caplog.text + + +def test_index_plugin_update_is_reported(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("example-plugin", version="1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + + findings = PluginUpdates().check() + + assert findings == [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ] + + +def test_index_plugin_queries_the_package_index_with_the_configured_timeout( + monkeypatch, data_folder, one_plugin +): + one_plugin(FakeDistribution("example-plugin")) + requests_made = [] + + def get(url, **kwargs): + requests_made.append((url, kwargs)) + return FakeResponse(payload={"info": {"version": "1.0.0"}}) + + monkeypatch.setattr("mvt.common.updates.requests.get", get) + monkeypatch.setattr("mvt.common.updates.settings.NETWORK_TIMEOUT", 3) + + PluginUpdates().check() + + assert requests_made == [ + ("https://pypi.org/pypi/example-plugin/json", {"timeout": 3}) + ] + + +def test_up_to_date_index_plugin_is_not_reported(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("example-plugin", version="1.2.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + + assert PluginUpdates().check() == [] + + +def test_unpublished_plugin_is_skipped_silently(monkeypatch, data_folder, one_plugin): + one_plugin(FakeDistribution("private-plugin")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(status_code=404), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_following_a_branch_is_reported( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls), + ) + monkeypatch.delenv("GIT_SSH_COMMAND", raising=False) + + findings = PluginUpdates().check() + + assert findings == [ + { + "name": "example-plugin", + "installed": "aaaaaaaa", + "latest": "bbbbbbbb", + "origin": "git", + "upgrade_command": ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@main'" + ), + } + ] + command, options = calls[0] + assert command == ["git", "ls-remote", REPOSITORY_URL, "main"] + assert options["env"]["GIT_TERMINAL_PROMPT"] == "0" + # ssh asks the terminal for a passphrase or a host key unless it is told + # not to, which git itself cannot prevent. + assert options["env"]["GIT_SSH_COMMAND"] == ( + "ssh -o BatchMode=yes -o ConnectTimeout=10" + ) + + +def test_batch_mode_options_come_before_the_configured_ssh_options( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls), + ) + monkeypatch.setenv("GIT_SSH_COMMAND", "ssh -o BatchMode=no -i /home/analyst/key") + + PluginUpdates().check() + + # ssh uses the first value it is given for a keyword, so an analyst asking + # for prompts cannot bring them back, while their other options still + # apply. + assert calls[0][1]["env"]["GIT_SSH_COMMAND"] == ( + "ssh -o BatchMode=yes -o ConnectTimeout=10 -o BatchMode=no -i /home/analyst/key" + ) + + +def test_repository_plugin_without_a_revision_follows_the_default_branch( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution()) + calls = [] + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\tHEAD\n", calls=calls), + ) + + findings = PluginUpdates().check() + + assert calls[0][0] == ["git", "ls-remote", REPOSITORY_URL, "HEAD"] + assert findings[0]["upgrade_command"] == ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}'" + ) + + +def test_hostile_revision_cannot_inject_into_the_upgrade_command( + monkeypatch, data_folder, one_plugin +): + revision = "main$(id)`id`;id" + one_plugin(_git_distribution(requested_revision=revision)) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/{revision}\n"), + ) + + upgrade_command = PluginUpdates().check()[0]["upgrade_command"] + + # Single quotes are the only quoting a shell does not expand anything in. + assert upgrade_command == ( + f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@{revision}'" + ) + assert shlex.split(upgrade_command) == [ + "pip", + "install", + "-U", + f"example-plugin @ git+{REPOSITORY_URL}@{revision}", + ] + + +def test_repository_plugin_with_an_option_like_url_is_skipped( + monkeypatch, data_folder, one_plugin +): + one_plugin( + FakeDistribution( + "example-plugin", + direct_url={ + "url": "--upload-pack=touch /tmp/mvt", + "vcs_info": {"vcs": "git", "commit_id": INSTALLED_COMMIT}, + }, + ) + ) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_repository_plugin_with_an_option_like_revision_is_skipped( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="--upload-pack=touch /tmp/mvt")) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_repository_plugin_at_the_latest_commit_is_not_reported( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{INSTALLED_COMMIT}\trefs/heads/main\n"), + ) + + assert PluginUpdates().check() == [] + + +def test_commit_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT)) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_short_commit_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT[:10])) + calls = [] + monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls)) + + assert PluginUpdates().check() == [] + assert calls == [] + + +def test_tag_pinned_repository_plugin_is_never_outdated( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="v1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout=f"{REMOTE_COMMIT}\trefs/tags/v1.0.0\n"), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_is_skipped_without_git(monkeypatch, data_folder, one_plugin): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(exception=FileNotFoundError("git")), + ) + + assert PluginUpdates().check() == [] + + +def test_repository_plugin_is_skipped_when_git_fails( + monkeypatch, data_folder, one_plugin +): + one_plugin(_git_distribution(requested_revision="main")) + monkeypatch.setattr( + "mvt.common.updates.subprocess.run", + _fake_git(stdout="", returncode=128), + ) + + assert PluginUpdates().check() == [] + + +def test_local_plugin_install_is_skipped(monkeypatch, data_folder, one_plugin): + one_plugin( + FakeDistribution( + "example-plugin", + direct_url={ + "url": "file:///home/analyst/example-plugin", + "dir_info": {"editable": True}, + }, + ) + ) + + def fail(*args, **kwargs): + raise AssertionError("a local plugin install must not be checked") + + monkeypatch.setattr("mvt.common.updates.requests.get", fail) + monkeypatch.setattr("mvt.common.updates.subprocess.run", fail) + + assert PluginUpdates().check() == [] + + +def test_check_stores_the_findings_and_the_check_timestamp( + monkeypatch, data_folder, one_plugin +): + one_plugin(FakeDistribution("example-plugin", version="1.0.0")) + monkeypatch.setattr( + "mvt.common.updates.requests.get", + lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}), + ) + plugin_updates = PluginUpdates() + + findings = plugin_updates.check() + + assert json.loads((data_folder / "plugin_updates.json").read_text()) == findings + assert (data_folder / "latest_plugins_check").read_text().isdigit() + assert PluginUpdates().get_findings() == findings + + +def test_findings_are_empty_before_the_first_check(data_folder): + assert PluginUpdates().get_findings() == [] + + +def test_malformed_cached_findings_are_dropped(data_folder): + plugin_updates = PluginUpdates() + usable = { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text( + json.dumps( + [ + {"oops": 1}, + "not a finding", + {"name": "half-plugin", "installed": "1.0.0"}, + {**usable, "latest": None}, + usable, + ] + ), + encoding="utf-8", + ) + + assert plugin_updates.get_findings() == [usable] + + +def test_corrupt_cached_findings_are_ignored(data_folder): + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text("{ not json", encoding="utf-8") + + assert PluginUpdates().get_findings() == [] + + +def test_cached_findings_of_upgraded_and_removed_plugins_are_dropped( + monkeypatch, data_folder +): + findings = [ + { + "name": "upgraded-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U upgraded-plugin", + }, + { + "name": "removed-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U removed-plugin", + }, + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + }, + ] + plugin_updates = PluginUpdates() + plugin_updates.set_findings(findings) + monkeypatch.setattr( + "mvt.common.updates.installed_plugin_distributions", + lambda: [ + # The analyst upgraded this plugin since the latest check. + FakeDistribution("upgraded-plugin", version="1.2.0"), + FakeDistribution("example-plugin", version="1.0.0"), + ], + ) + + assert plugin_updates.current_findings() == [findings[2]] + + +def test_cached_findings_of_updated_repository_plugins_are_dropped(data_folder): + findings = [ + { + "name": "example-plugin", + "installed": "aaaaaaaa", + "latest": "bbbbbbbb", + "origin": "git", + "upgrade_command": "pip install -U example-plugin", + } + ] + plugin_updates = PluginUpdates() + plugin_updates.set_findings(findings) + + assert plugin_updates.current_findings([_git_distribution()]) == findings + assert ( + plugin_updates.current_findings([_git_distribution(commit=REMOTE_COMMIT)]) == [] + ) + + +def test_corrupt_check_timestamp_does_not_raise(data_folder): + plugin_updates = PluginUpdates() + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "latest_plugins_check").write_text("truncated", encoding="utf-8") + + assert plugin_updates.get_latest_check() == 0 + assert plugin_updates.should_check() == (True, 0) + + +def test_should_check_is_throttled_for_twelve_hours(data_folder): + plugin_updates = PluginUpdates() + plugin_updates.set_findings([]) + + recent = datetime.now() - timedelta(hours=4) + with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(int(recent.timestamp()))) + + should_check, hours = plugin_updates.should_check() + assert not should_check + assert hours == 8 + + old = datetime.now() - timedelta(hours=13) + with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle: + handle.write(str(int(old.timestamp()))) + + assert plugin_updates.should_check() == (True, 0) + + +def test_should_check_without_a_previous_check(data_folder): + assert PluginUpdates().should_check() == (True, 0) + + +@pytest.fixture +def no_version_check(monkeypatch): + monkeypatch.setattr(MVTUpdates, "check", lambda self: "") + # Keep rich from wrapping the plugin lines while they are being asserted. + monkeypatch.setenv("COLUMNS", "200") + + +@pytest.fixture +def throttled_cache(monkeypatch, data_folder): + """Fill the findings cache and put the check inside its throttle window.""" + + def fill(findings, distributions): + PluginUpdates().set_findings(findings) + monkeypatch.setattr( + logo, "installed_plugin_distributions", lambda: distributions + ) + monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8)) + monkeypatch.setattr( + PluginUpdates, + "check", + lambda self: pytest.fail("the check must be throttled"), + ) + + return fill + + +def test_logo_prints_the_cached_plugin_updates( + capsys, no_version_check, throttled_cache +): + throttled_cache( + [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ], + [FakeDistribution("example-plugin", version="1.0.0")], + ) + + logo.check_updates(disable_indicator_check=True) + + output = capsys.readouterr().out + assert "Plugin updates available:" in output + assert "example-plugin 1.0.0 → 1.2.0 (pip install -U example-plugin)" in output + + +def test_logo_does_not_print_a_cached_update_of_an_upgraded_plugin( + capsys, no_version_check, throttled_cache +): + throttled_cache( + [ + { + "name": "example-plugin", + "installed": "1.0.0", + "latest": "1.2.0", + "origin": "pypi", + "upgrade_command": "pip install -U example-plugin", + } + ], + # The analyst already upgraded the plugin the cached finding is about. + [FakeDistribution("example-plugin", version="1.2.0")], + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_prints_nothing_when_throttled_without_findings( + capsys, no_version_check, throttled_cache +): + throttled_cache([], [FakeDistribution("example-plugin")]) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_survives_a_corrupt_plugin_cache( + monkeypatch, capsys, data_folder, no_version_check +): + data_folder.mkdir(parents=True, exist_ok=True) + (data_folder / "plugin_updates.json").write_text( + json.dumps([{"oops": 1}]), encoding="utf-8" + ) + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: [FakeDistribution("example-plugin")], + ) + monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8)) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_without_plugins( + monkeypatch, capsys, no_version_check +): + monkeypatch.setattr(logo, "installed_plugin_distributions", list) + monkeypatch.setattr( + PluginUpdates, + "should_check", + lambda self: pytest.fail("plugins must not be checked without plugins"), + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_without_network_access( + monkeypatch, capsys, no_version_check +): + monkeypatch.setattr("mvt.common.logo.settings.NETWORK_ACCESS_ALLOWED", False) + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: pytest.fail("plugins must not be listed without network access"), + ) + + logo.check_updates(disable_indicator_check=True) + + assert "Plugin updates" not in capsys.readouterr().out + + +def test_logo_skips_the_plugin_check_when_update_checks_are_disabled( + monkeypatch, capsys +): + monkeypatch.setattr( + logo, + "installed_plugin_distributions", + lambda: pytest.fail("plugins must not be checked with --disable-update-check"), + ) + + logo.check_updates(disable_version_check=True, disable_indicator_check=True) + + assert capsys.readouterr().out == "" diff --git a/tests/common/test_utils.py b/tests/common/test_utils.py index 4dbe5c0..6bef147 100644 --- a/tests/common/test_utils.py +++ b/tests/common/test_utils.py @@ -8,6 +8,7 @@ import logging import os from datetime import datetime +from mvt.common.log import MVTLogHandler from mvt.common.utils import ( CustomJSONEncoder, convert_datetime_to_iso, @@ -16,6 +17,8 @@ from mvt.common.utils import ( convert_unix_to_utc_datetime, generate_hashes_from_path, get_sha256_from_file_path, + init_logging, + set_verbose_logging, ) from ..utils import get_artifact_folder @@ -103,3 +106,46 @@ class TestCustomJSONEncoder: json.dumps({"name": "家".encode()}, cls=CustomJSONEncoder) == '{"name": "\\u5bb6"}' ) + + +class TestInitLogging: + def test__init_logging_is_idempotent(self): + # Loaded module packages may import an MVT CLI module, which calls + # init_logging() again at import time. A second call must not add + # a duplicate console handler. + log = logging.getLogger("mvt") + init_logging() + handler_count = sum( + isinstance(handler, MVTLogHandler) for handler in log.handlers + ) + init_logging() + assert ( + sum(isinstance(handler, MVTLogHandler) for handler in log.handlers) + == handler_count + ) + + def test_verbose_logging_finds_the_console_handler_among_others(self): + # Something else may have attached a handler to the "mvt" logger + # before MVT did, so the console handler is not always the first. + log = logging.getLogger("mvt") + init_logging() + foreign_handler = logging.NullHandler() + foreign_handler.setLevel(logging.CRITICAL) + log.handlers.insert(0, foreign_handler) + + try: + set_verbose_logging(True) + console_handlers = [ + handler + for handler in log.handlers + if isinstance(handler, MVTLogHandler) + ] + assert console_handlers + assert all(handler.level == logging.DEBUG for handler in console_handlers) + assert foreign_handler.level == logging.CRITICAL + + set_verbose_logging(False) + assert all(handler.level == logging.INFO for handler in console_handlers) + assert foreign_handler.level == logging.CRITICAL + finally: + log.handlers.remove(foreign_handler) diff --git a/tests/conftest.py b/tests/conftest.py index c89f629..06a890a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,11 @@ import os import pytest +from mvt.common.cli_plugins import ( + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, +) from mvt.common.indicators import Indicators from .artifacts.generate_stix import generate_test_stix_file @@ -58,3 +63,33 @@ def indicators_factory(indicator_file): return ind return f + + +@pytest.fixture() +def restore_cli_commands(monkeypatch): + """Keep the external commands a test registers out of the next test. + + Each CLI group is a module-level object shared by every test, so a test + registering plugin or environment commands on one has to put it back. The + groups are imported here rather than at the top of the file, so that + collecting the tests does not import three CLIs for the sake of one + fixture. + """ + from mvt.android.cli import cli as android_cli + from mvt.cli import cli as neutral_cli + from mvt.ios.cli import cli as ios_cli + + groups = (neutral_cli, ios_cli, android_cli) + for variable in ( + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ): + monkeypatch.delenv(variable, raising=False) + originals = [dict(group.commands) for group in groups] + yield + for group, commands in zip(groups, originals): + group.commands.clear() + group.commands.update(commands) + if hasattr(group, "_mvt_external_command_sources"): + delattr(group, "_mvt_external_command_sources") diff --git a/tests/ios_backup/test_interactionc.py b/tests/ios_backup/test_interactionc.py new file mode 100644 index 0000000..eec2656 --- /dev/null +++ b/tests/ios_backup/test_interactionc.py @@ -0,0 +1,105 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.common.module import run_module +from mvt.ios.modules.mixed.interactionc import InteractionC +from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts + +from ..utils import get_ios_backup_folder + + +class TestInteractionCModule: + def test_extraction_with_whatsapp_contacts(self): + contacts = WhatsappContacts(target_path=get_ios_backup_folder()) + run_module(contacts) + + m = InteractionC(target_path=get_ios_backup_folder()) + m.dependency_modules = {WhatsappContacts: contacts} + run_module(m) + + assert len(m.results) == 3 + + incoming = next( + r for r in m.results if r["sender_identifier"] == "100000000000001@lid" + ) + assert incoming["direction"] == "INCOMING" + assert incoming["sender_resolved_phone_number"] == "+14155550100" + assert incoming["sender_resolved_name"] == "Alice Example" + + outgoing = next( + r for r in m.results if r["direction"] == "OUTGOING" + ) + assert outgoing["recipient_identifier"] == "+14155550100" + assert outgoing["recipient_resolved_name"] == "Alice Example" + assert outgoing["domain_resolved_phone_number"] == "+14155550100" + assert outgoing["domain_resolved_name"] == "Alice Example" + + sms = next( + r for r in m.results if r["bundle_id"] == "com.apple.MobileSMS" + ) + assert sms.get("sender_resolved_name") is None + assert sms["sender_display_name"] == "Bob Example" + + events = [entry["data"] for entry in m.timeline] + assert ( + "[net.whatsapp.WhatsApp] INCOMING from " + "Alice Example (+14155550100) to local user" in events + ) + assert ( + "[net.whatsapp.WhatsApp] OUTGOING from local user to " + "Alice Example (+14155550100)" in events + ) + assert ( + "[com.apple.MobileSMS] INCOMING from " + "Bob Example (+14155550101) to local user" in events + ) + + # The creation date is only serialized when it diverges from the + # start date; the SMS record was created 90 days after the event. + creation_events = [ + entry + for entry in m.timeline + if entry["event"] == "interactions_creation_date" + ] + assert len(creation_events) == 1 + assert creation_events[0]["timestamp"] == "2025-12-09 12:26:40.000000" + assert creation_events[0]["data"] == ( + "Interaction record created 90 days after the event: " + "[com.apple.MobileSMS] INCOMING from " + "Bob Example (+14155550101) to local user" + ) + + # Per-contact aggregate dates use contact-centric data strings. + first_seen = [ + entry + for entry in m.timeline + if entry["event"] == "first_incoming_sender_date" + ] + assert len(first_seen) == 1 + assert first_seen[0]["timestamp"] == "2025-09-03 13:46:40.000000" + assert first_seen[0]["data"] == ( + "First incoming interaction from Bob Example (+14155550101)" + ) + assert ( + "Last incoming interaction from Bob Example (+14155550101)" + in events + ) + + def test_extraction_without_whatsapp_contacts(self): + # Without the WhatsappContacts dependency the module still runs, and + # unresolvable LIDs are shown as-is. + m = InteractionC(target_path=get_ios_backup_folder()) + run_module(m) + + assert len(m.results) == 3 + events = [entry["data"] for entry in m.timeline] + assert ( + "[net.whatsapp.WhatsApp] INCOMING from " + "100000000000001@lid to local user" in events + ) + assert ( + "[net.whatsapp.WhatsApp] OUTGOING from local user to " + "+14155550100" in events + ) diff --git a/tests/ios_backup/test_whatsapp.py b/tests/ios_backup/test_whatsapp.py index 864fb84..3bc920d 100644 --- a/tests/ios_backup/test_whatsapp.py +++ b/tests/ios_backup/test_whatsapp.py @@ -6,8 +6,79 @@ import logging from mvt.common.indicators import Indicators +from mvt.common.module import run_module from mvt.ios.modules.mixed.whatsapp import Whatsapp +from ..utils import get_ios_backup_folder + + +def test_extraction(): + m = Whatsapp(target_path=get_ios_backup_folder()) + run_module(m) + + messages = [r for r in m.results if "ZTEXT" in r] + sessions = [r for r in m.results if r.get("record_type") == "chat_session"] + pairs = [ + r for r in m.results + if r.get("record_type") == "lid_phone_number_pair" + ] + assert len(messages) == 3 + assert len(sessions) == 2 + assert len(pairs) == 1 + + assert pairs[0]["lid"] == "100000000000001" + assert pairs[0]["phone_number"] == "14155550100" + assert pairs[0]["pair_timestamp"] == "2025-08-25 07:33:20.000000" + + linked = next(r for r in messages if r.get("links")) + assert linked["links"] == ["https://example.org/news"] + + alice = next(s for s in sessions if s["partner_name"] == "Alice Example") + assert alice["contact_jid"] == "100000000000001@lid" + assert alice["partner_resolved_phone_number"] == "+14155550100" + assert alice["first_stored_message_date"] == "2025-08-27 15:06:40.000000" + assert alice["last_message_date"] == "2025-08-28 18:53:20.000000" + assert alice["group_creation_date"] is None + assert alice["stored_message_count"] == 2 + + group = next(s for s in sessions if s["partner_name"] == "Example Group") + assert group["group_creation_date"] == "2025-08-21 20:13:20.000000" + assert group["first_stored_message_date"] == "2025-08-29 22:40:00.000000" + # The last stored message predates the session's own last-message date: + # the newest message in this chat was deleted. + assert group["last_stored_message_date"] == "2025-08-29 22:40:00.000000" + assert group["last_message_date"] == "2025-08-31 02:26:40.000000" + + # 3 message events, first/last per chat, the group creation and the + # LID-phone number pair. + assert len(m.timeline) == 9 + events = { + (entry["event"], entry["timestamp"]): entry["data"] + for entry in m.timeline + } + # Alice's session is keyed by LID but labelled with the phone number + # resolved through LID.sqlite. + assert events[("chat_first_message", "2025-08-27 15:06:40.000000")] == ( + "First stored message in WhatsApp chat with " + "'Alice Example' (+14155550100)" + ) + assert events[("chat_last_message", "2025-08-28 18:53:20.000000")] == ( + "Last message in WhatsApp chat with " + "'Alice Example' (+14155550100)" + ) + assert events[("lid_pair_recorded", "2025-08-25 07:33:20.000000")] == ( + "WhatsApp associated LID 100000000000001 with " + "phone number 14155550100" + ) + assert events[("group_created", "2025-08-21 20:13:20.000000")] == ( + "WhatsApp group chat 'Example Group' " + "(120000000000000001@g.us) was created" + ) + assert ("chat_first_message", "2025-08-29 22:40:00.000000") in events + assert ("chat_last_message", "2025-08-31 02:26:40.000000") in events + + assert len(m.alertstore.alerts) == 0 + def test_collect_url_results_includes_expansion(): module = Whatsapp( diff --git a/tests/ios_backup/test_whatsapp_contacts.py b/tests/ios_backup/test_whatsapp_contacts.py new file mode 100644 index 0000000..a91b8f1 --- /dev/null +++ b/tests/ios_backup/test_whatsapp_contacts.py @@ -0,0 +1,83 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.common.module import run_module +from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts + +from ..utils import get_ios_backup_folder + + +class TestWhatsappContactsModule: + def test_extraction(self): + m = WhatsappContacts(target_path=get_ios_backup_folder()) + run_module(m) + assert len(m.results) == 2 + + alice = next(r for r in m.results if r["given_name"] == "Alice") + assert alice["full_name"] == "Alice Example" + assert alice["phone_number"] == "+14155550100" + assert alice["whatsapp_id"] == "14155550100@s.whatsapp.net" + assert alice["lid"] == "100000000000001@lid" + assert alice["user_name"] == "alice.example" + assert alice["disappearing_mode_duration"] == 86400.0 + assert alice["disappearing_mode_is_on"] is True + assert alice["disappearing_mode_label"] == "24 hours" + assert alice["disappearing_mode_timestamp"] == "2025-07-23 21:46:40.000000" + assert alice["about_timestamp"] == "2025-07-12 08:00:00.000000" + assert alice["about_expiration_timestamp"] == "2025-08-16 01:20:00.000000" + assert alice["last_updated"] == "2025-08-04 11:33:20.000000" + + bob = next(r for r in m.results if r["given_name"] == "Bob") + assert bob["lid"] is None + assert bob["disappearing_mode_duration"] is None + assert bob["disappearing_mode_is_on"] is False + assert bob["disappearing_mode_label"] == "off" + assert bob["disappearing_mode_timestamp"] is None + + # Alice: disappearing_mode_set, about_changed, about_expiration and + # contact_last_updated. Bob: contact_last_updated only. + assert len(m.timeline) == 5 + + events = { + (entry["event"], entry["timestamp"]): entry["data"] + for entry in m.timeline + } + assert ( + "24 hours" + in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")] + ) + assert ( + "14155550100@s.whatsapp.net (Alice Example)" + in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")] + ) + assert ( + 'changed to "Hey there! I am using WhatsApp."' + in events[("about_changed", "2025-07-12 08:00:00.000000")] + ) + assert ( + "scheduled to expire" + in events[("about_expiration", "2025-08-16 01:20:00.000000")] + ) + + updated = [ + entry["data"] + for entry in m.timeline + if entry["event"] == "contact_last_updated" + ] + assert len(updated) == 2 + assert all( + entry["timestamp"] == "2025-08-04 11:33:20.000000" + for entry in m.timeline + if entry["event"] == "contact_last_updated" + ) + assert any("14155550101@s.whatsapp.net (Bob Example)" in d for d in updated) + + assert len(m.alertstore.alerts) == 0 + + def test_missing_database(self, tmp_path): + m = WhatsappContacts(target_path=str(tmp_path)) + run_module(m) + assert m.results == [] + assert len(m.alertstore.alerts) == 0 diff --git a/tests/ios_fs/test_filesystem.py b/tests/ios_fs/test_filesystem.py index 9fa664f..636c004 100644 --- a/tests/ios_fs/test_filesystem.py +++ b/tests/ios_fs/test_filesystem.py @@ -15,8 +15,8 @@ class TestFilesystem: def test_filesystem(self): m = Filesystem(target_path=get_ios_backup_folder()) run_module(m) - assert len(m.results) == 15 - assert len(m.timeline) == 15 + assert len(m.results) == 23 + assert len(m.timeline) == 23 assert len(m.alertstore.alerts) == 0 def test_detection(self, indicator_file): @@ -29,6 +29,6 @@ class TestFilesystem: ) m.indicators = ind run_module(m) - assert len(m.results) == 15 - assert len(m.timeline) == 15 + assert len(m.results) == 23 + assert len(m.timeline) == 23 assert len(m.alertstore.alerts) == 1 diff --git a/tests/plugin_fixtures.py b/tests/plugin_fixtures.py new file mode 100644 index 0000000..ef989dc --- /dev/null +++ b/tests/plugin_fixtures.py @@ -0,0 +1,86 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +"""Helpers building throwaway plugin distributions for the tests. + +Some plugin behaviour only shows up in a fresh interpreter: what an import +executes, and what a plugin sees when MVT is imported before or after it. +These helpers write an importable distribution with a real entry point and +run a script against it in a subprocess, with a temporary home so that the +subprocess cannot touch the configuration of whoever runs the tests. +""" + +import os +import subprocess +import sys +from pathlib import Path +from typing import Optional + +FIXTURE_COMMAND_NAME = "fixture-plugin" +FIXTURE_MODULE_NAME = "fixture_cli_plugin" +FIXTURE_DISTRIBUTION_NAME = "fixture-cli-plugin" + + +def write_cli_plugin_distribution( + site_path: Path, + entry_point_group: str, + module_source: str, +) -> Path: + """Write a distribution registering a CLI plugin entry point. + + :param site_path: Folder to write the distribution into, to be added to + the import path of the interpreter loading it. + :param entry_point_group: Entry-point group to register the command in. + :param module_source: Source of the plugin module, which must define a + Click command named `cli`. + :returns: The folder the distribution was written to. + """ + site_path.mkdir(parents=True, exist_ok=True) + (site_path / f"{FIXTURE_MODULE_NAME}.py").write_text( + module_source, encoding="utf-8" + ) + + dist_info = ( + site_path / f"{FIXTURE_DISTRIBUTION_NAME.replace('-', '_')}-1.0.dist-info" + ) + dist_info.mkdir(exist_ok=True) + (dist_info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {FIXTURE_DISTRIBUTION_NAME}\nVersion: 1.0\n", + encoding="utf-8", + ) + (dist_info / "entry_points.txt").write_text( + f"[{entry_point_group}]\n{FIXTURE_COMMAND_NAME} = {FIXTURE_MODULE_NAME}:cli\n", + encoding="utf-8", + ) + return site_path + + +def run_isolated_python( + script: str, + home: Path, + site_path: Optional[Path] = None, + **environment: str, +) -> subprocess.CompletedProcess: + """Run a script in a fresh interpreter with its own configuration folder. + + Importing MVT writes its configuration file, so the subprocess gets a + temporary home and no MVT environment variables from the test session. + """ + isolated_environment = { + key: value for key, value in os.environ.items() if not key.startswith("MVT_") + } + isolated_environment["HOME"] = str(home) + isolated_environment["XDG_CONFIG_HOME"] = str(home / "config") + isolated_environment["XDG_DATA_HOME"] = str(home / "data") + if site_path is not None: + isolated_environment["PYTHONPATH"] = str(site_path) + isolated_environment.update(environment) + + return subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env=isolated_environment, + ) diff --git a/tests/test_check_android_androidqf.py b/tests/test_check_android_androidqf.py index 2253a50..4e6ef98 100644 --- a/tests/test_check_android_androidqf.py +++ b/tests/test_check_android_androidqf.py @@ -155,7 +155,7 @@ class TestCheckAndroidqfCommand: result = runner.invoke(check_androidqf, [str(path)]) assert result.exit_code == 0 - assert "Skipping backup modules as backup.ab is malformed" in caplog.text + assert "Skipping backup modules: Invalid backup format" in caplog.text assert not any( record.levelname in {"CRITICAL", "FATAL"} for record in caplog.records ) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6495243 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,48 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from click.testing import CliRunner + +from mvt.cli import cli +from mvt.common.updates import IndicatorsUpdates +from mvt.common.version import MVT_VERSION + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + + +class TestMvtCommand: + def test_running_mvt_alone_shows_the_logo_and_the_commands(self): + result = CliRunner().invoke(cli, OFFLINE) + + assert result.exit_code == 0 + logo_at = result.output.index("Mobile Verification Toolkit") + usage_at = result.output.index("Usage:") + assert logo_at < usage_at + assert "mvt-ios" in result.output and "mvt-android" in result.output + + def test_help_reminds_where_the_analysis_runs(self): + result = CliRunner().invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "mvt-ios" in result.output + assert "mvt-android" in result.output + + def test_version_prints_the_installed_version(self): + result = CliRunner().invoke(cli, [*OFFLINE, "version"]) + + assert result.exit_code == 0 + assert f"Version: {MVT_VERSION}" in result.output + + def test_download_iocs_updates_the_indicators(self, monkeypatch): + updates = [] + monkeypatch.setattr( + IndicatorsUpdates, "update", lambda self: updates.append(self) + ) + + result = CliRunner().invoke(cli, [*OFFLINE, "download-iocs"]) + + assert result.exit_code == 0 + assert len(updates) == 1 diff --git a/tests/test_cli_entry_points.py b/tests/test_cli_entry_points.py new file mode 100644 index 0000000..1b51413 --- /dev/null +++ b/tests/test_cli_entry_points.py @@ -0,0 +1,280 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import sys +from types import SimpleNamespace + +import click +import pytest + +import mvt.android +import mvt.cli +import mvt.ios +from mvt.android.cli import cli as android_cli +from mvt.android.cli import main as android_main +from mvt.cli import cli as mvt_cli +from mvt.cli import main as mvt_main +from mvt.common.cli_plugins import ( + ANDROID_CLI_PLUGIN_GROUP, + IOS_CLI_PLUGIN_GROUP, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + MVT_CUSTOM_COMMANDS_ENV, + MVT_IOS_CUSTOM_COMMANDS_ENV, + NEUTRAL_CLI_PLUGIN_GROUP, +) +from mvt.ios.cli import cli as ios_cli +from mvt.ios.cli import main as ios_main + +from .plugin_fixtures import ( + FIXTURE_COMMAND_NAME, + run_isolated_python, + write_cli_plugin_distribution, +) + +MARKER_PLUGIN_TEMPLATE = """ +import os + +import click + +# Touched when this module is imported, so a test can tell whether loading MVT +# executed the plugin. +open(os.environ["FIXTURE_PLUGIN_MARKER"], "a").close() + + +@click.command() +def cli(): + click.echo("fixture plugin ran") +""" + +PROGRAMS = { + "mvt": (mvt.cli, mvt_cli, NEUTRAL_CLI_PLUGIN_GROUP, MVT_CUSTOM_COMMANDS_ENV), + "mvt-ios": (mvt.ios, ios_cli, IOS_CLI_PLUGIN_GROUP, MVT_IOS_CUSTOM_COMMANDS_ENV), + "mvt-android": ( + mvt.android, + android_cli, + ANDROID_CLI_PLUGIN_GROUP, + MVT_ANDROID_CUSTOM_COMMANDS_ENV, + ), +} + +CASE_SUMMARY_COMMAND = """ +import click + + +@click.command("case-summary") +def cli(): + click.echo("case summary ran") +""" + +# The entry-point group of another program, for each program: no group may add +# its commands to a CLI other than its own. +OTHER_PROGRAMS_GROUP = { + "mvt": IOS_CLI_PLUGIN_GROUP, + "mvt-ios": NEUTRAL_CLI_PLUGIN_GROUP, + "mvt-android": NEUTRAL_CLI_PLUGIN_GROUP, +} + + +def _install_fixture_entry_point(monkeypatch, entry_point_group, command): + def entry_points(*, group): + if group != entry_point_group: + return [] + return [ + SimpleNamespace( + name=FIXTURE_COMMAND_NAME, + value="fixture_cli_plugin:cli", + load=lambda: command, + dist=SimpleNamespace( + metadata={"Name": "fixture-cli-plugin"}, version="1.0" + ), + ) + ] + + monkeypatch.setattr( + "mvt.common.cli_plugins.importlib.metadata.entry_points", entry_points + ) + + +def _offline_argv(program, *arguments): + """Build an argument list which keeps the CLI from checking for updates.""" + return [ + program, + "--disable-update-check", + "--disable-indicator-update-check", + *arguments, + ] + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_registers_installed_plugins_before_running_the_cli( + program, monkeypatch, capsys, restore_cli_commands +): + package, group, entry_point_group, _ = PROGRAMS[program] + + @click.command() + def fixture_command(): + click.echo("fixture plugin ran") + + _install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command) + monkeypatch.setattr(sys, "argv", _offline_argv(program, FIXTURE_COMMAND_NAME)) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "fixture plugin ran" in capsys.readouterr().out + assert FIXTURE_COMMAND_NAME in group.commands + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_completes_plugin_command_names( + program, monkeypatch, capsys, restore_cli_commands +): + package, _, entry_point_group, _ = PROGRAMS[program] + + @click.command() + def fixture_command(): + pass + + _install_fixture_entry_point(monkeypatch, entry_point_group, fixture_command) + complete_variable = f"_{program.upper().replace('-', '_')}_COMPLETE" + monkeypatch.setenv(complete_variable, "bash_complete") + monkeypatch.setenv("COMP_WORDS", f"{program} fixture") + monkeypatch.setenv("COMP_CWORD", "1") + monkeypatch.setattr(sys, "argv", [program]) + + with pytest.raises(SystemExit): + package.main() + + assert f"plain,{FIXTURE_COMMAND_NAME}" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_still_loads_commands_from_a_file( + program, monkeypatch, capsys, tmp_path, restore_cli_commands +): + package, _, entry_point_group, _ = PROGRAMS[program] + command_path = tmp_path / "case_summary.py" + command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8") + _install_fixture_entry_point( + monkeypatch, entry_point_group, click.Command("unused") + ) + monkeypatch.setattr( + sys, + "argv", + _offline_argv(program, "--load-command", str(command_path), "case-summary"), + ) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "case summary ran" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_loads_commands_from_the_environment_variable( + program, monkeypatch, capsys, tmp_path, restore_cli_commands +): + # Each CLI reads its own variable, so a main() reading another CLI's would + # go unnoticed without this. + package, _, _, environment_variable = PROGRAMS[program] + command_path = tmp_path / "case_summary.py" + command_path.write_text(CASE_SUMMARY_COMMAND, encoding="utf-8") + monkeypatch.setenv(environment_variable, str(command_path)) + monkeypatch.setattr(sys, "argv", _offline_argv(program, "case-summary")) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert "case summary ran" in capsys.readouterr().out + + +@pytest.mark.parametrize("program", sorted(PROGRAMS)) +def test_main_ignores_the_entry_point_groups_of_the_other_programs( + program, monkeypatch, capsys, restore_cli_commands +): + package, group, _, _ = PROGRAMS[program] + _install_fixture_entry_point( + monkeypatch, + OTHER_PROGRAMS_GROUP[program], + click.Command(FIXTURE_COMMAND_NAME), + ) + monkeypatch.setattr(sys, "argv", _offline_argv(program, "--help")) + + with pytest.raises(SystemExit) as exit_info: + package.main() + + assert exit_info.value.code == 0 + assert FIXTURE_COMMAND_NAME not in group.commands + assert FIXTURE_COMMAND_NAME not in capsys.readouterr().out + + +def test_the_console_script_targets_are_importable(): + # [project.scripts] points at these, so they must stay where they are. + assert mvt.cli.main is mvt_main + assert mvt.ios.main is ios_main + assert mvt.android.main is android_main + + +def test_importing_mvt_does_not_import_a_cli(tmp_path): + # The mvt package deliberately re-exports nothing of mvt.cli, so that + # importing MVT stays cheap and free of side effects. + result = run_isolated_python( + "import sys\n" + "import mvt\n" + "print('imported a cli' if 'mvt.cli' in sys.modules else 'imported mvt')\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "imported mvt" + + +def test_importing_mvt_does_not_run_installed_plugins(tmp_path): + site_path = write_cli_plugin_distribution( + tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE + ) + marker = tmp_path / "plugin-imported" + + result = run_isolated_python( + "import mvt.ios.cli\nimport mvt.android.cli\nprint('imported')", + home=tmp_path / "home", + site_path=site_path, + FIXTURE_PLUGIN_MARKER=str(marker), + ) + + assert result.returncode == 0, result.stderr + assert "imported" in result.stdout + assert not marker.exists() + + +def test_registering_the_plugins_runs_the_entry_point(tmp_path): + site_path = write_cli_plugin_distribution( + tmp_path / "site", IOS_CLI_PLUGIN_GROUP, MARKER_PLUGIN_TEMPLATE + ) + marker = tmp_path / "plugin-imported" + + result = run_isolated_python( + "import click\n" + "from mvt.common.cli_plugins import (\n" + " IOS_CLI_PLUGIN_GROUP,\n" + " BrokenPluginCommand,\n" + " register_installed_cli_commands,\n" + ")\n" + "group = click.Group()\n" + "register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)\n" + f"command = group.commands[{FIXTURE_COMMAND_NAME!r}]\n" + "assert not isinstance(command, BrokenPluginCommand), command.help\n" + "print('registered')\n", + home=tmp_path / "home", + site_path=site_path, + FIXTURE_PLUGIN_MARKER=str(marker), + ) + + assert result.returncode == 0, result.stderr + assert "registered" in result.stdout + assert marker.exists() diff --git a/tests/test_cli_startup.py b/tests/test_cli_startup.py new file mode 100644 index 0000000..e009256 --- /dev/null +++ b/tests/test_cli_startup.py @@ -0,0 +1,28 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import pytest + +from .plugin_fixtures import run_isolated_python + +# Importing a platform CLI must only build its command tree: the console +# scripts import it before Click can answer a shell completion request, which +# the completion scripts make on every keystroke. Every command imports what +# it runs when it is invoked. Each of these costs tens of milliseconds to +# import and is the sign that a command implementation is imported too early. +HEAVY_MODULES = ("pydantic", "requests", "Crypto", "mvt.common.module") + + +@pytest.mark.parametrize("cli_module", ("mvt.ios.cli", "mvt.android.cli")) +def test_importing_a_cli_does_not_import_the_module_machinery(cli_module, tmp_path): + result = run_isolated_python( + "import sys\n" + f"import {cli_module}\n" + f"print(','.join(name for name in {HEAVY_MODULES!r} if name in sys.modules))\n", + home=tmp_path / "home", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "", f"{cli_module} imported {result.stdout.strip()}" diff --git a/tests/test_cli_verbose.py b/tests/test_cli_verbose.py new file mode 100644 index 0000000..d8826ff --- /dev/null +++ b/tests/test_cli_verbose.py @@ -0,0 +1,107 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2026 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging + +import pytest +from click.testing import CliRunner + +from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli +from mvt.common.log import MVTLogHandler +from mvt.common.utils import set_verbose_logging +from mvt.ios.cli import cli as ios_cli + +# Keep the banner of the group callback from checking for updates online. +OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"] + +PROGRAMS = {"mvt": mvt_cli, "mvt-ios": ios_cli, "mvt-android": android_cli} + + +@pytest.fixture(autouse=True) +def _reset_console_level(): + """Leave the console handler at its default level after every test.""" + yield + set_verbose_logging(False) + + +def _console_level(): + """Return the level of MVT's own console log handler.""" + for handler in logging.getLogger("mvt").handlers: + if isinstance(handler, MVTLogHandler): + return handler.level + raise AssertionError("MVT has no console log handler") + + +class TestVerboseOnTheCommands: + @pytest.mark.parametrize("program", sorted(PROGRAMS)) + def test_verbose_before_the_command_name_turns_on_debug(self, program): + cli = PROGRAMS[program] + + result = CliRunner().invoke(cli, [*OFFLINE, "--verbose", "version"]) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + @pytest.mark.parametrize("program", sorted(PROGRAMS)) + def test_a_run_without_verbose_goes_back_to_info(self, program): + cli = PROGRAMS[program] + CliRunner().invoke(cli, [*OFFLINE, "--verbose", "version"]) + + result = CliRunner().invoke(cli, [*OFFLINE, "version"]) + + assert result.exit_code == 0 + assert _console_level() == logging.INFO + + def test_mvt_verbose_without_a_command_prints_the_help(self): + result = CliRunner().invoke(mvt_cli, [*OFFLINE, "--verbose"]) + + assert result.exit_code == 0 + assert "Usage:" in result.output + assert _console_level() == logging.DEBUG + + +class TestVerboseOnTheCheckCommands: + def test_ios_command_default_does_not_undo_the_cli_choice(self, tmp_path): + result = CliRunner().invoke( + ios_cli, + [*OFFLINE, "--verbose", "check-backup", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_ios_verbose_after_the_command_name_still_works(self, tmp_path): + result = CliRunner().invoke( + ios_cli, + [*OFFLINE, "check-backup", "--verbose", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_android_command_default_does_not_undo_the_cli_choice(self, tmp_path): + result = CliRunner().invoke( + android_cli, + [*OFFLINE, "--verbose", "check-bugreport", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_android_verbose_after_the_command_name_still_works(self, tmp_path): + result = CliRunner().invoke( + android_cli, + [*OFFLINE, "check-bugreport", "--verbose", "--list-modules", str(tmp_path)], + ) + + assert result.exit_code == 0 + assert _console_level() == logging.DEBUG + + def test_the_command_option_says_it_is_kept_for_compatibility(self): + result = CliRunner().invoke(ios_cli, [*OFFLINE, "check-backup", "--help"]) + + assert result.exit_code == 0 + assert "kept for compatibility" in result.output diff --git a/tests/test_completion.py b/tests/test_completion.py index 48c0177..08ad3ce 100644 --- a/tests/test_completion.py +++ b/tests/test_completion.py @@ -6,56 +6,60 @@ from click.testing import CliRunner from mvt.android.cli import cli as android_cli +from mvt.cli import cli as mvt_cli from mvt.ios.cli import cli as ios_cli class TestCompletionCommand: def test_completion_prints_instructions_by_default(self): runner = CliRunner() - result = runner.invoke(ios_cli, ["completion"]) + result = runner.invoke(mvt_cli, ["completion"]) assert result.exit_code == 0 - assert "Shell completion for mvt-ios" in result.output - assert "mvt-ios completion bash > ~/.mvt-ios-complete.bash" in result.output + assert "Shell completion for mvt, mvt-ios and mvt-android" in result.output + assert "mvt completion bash > ~/.mvt-complete.bash" in result.output assert "Mobile Verification Toolkit" not in result.output - def test_completion_prints_bash_script(self): + def test_completion_bash_script_covers_every_cli(self): runner = CliRunner() - result = runner.invoke(ios_cli, ["completion", "bash"]) + result = runner.invoke(mvt_cli, ["completion", "bash"]) assert result.exit_code == 0 + assert "_MVT_COMPLETE=bash_complete" in result.output assert "_MVT_IOS_COMPLETE=bash_complete" in result.output + assert "_MVT_ANDROID_COMPLETE=bash_complete" in result.output assert "complete -o nosort" in result.output - assert "mvt-ios" in result.output assert "Mobile Verification Toolkit" not in result.output - def test_completion_prints_fish_script(self): + def test_completion_fish_script_covers_every_cli(self): runner = CliRunner() - result = runner.invoke(android_cli, ["completion", "fish"]) + result = runner.invoke(mvt_cli, ["completion", "fish"]) assert result.exit_code == 0 - assert "_MVT_ANDROID_COMPLETE=fish_complete" in result.output + assert "complete --no-files --command mvt-ios" in result.output assert "complete --no-files --command mvt-android" in result.output + assert "complete --no-files --command mvt " in result.output assert "Mobile Verification Toolkit" not in result.output def test_completion_install_updates_bashrc_once(self, tmp_path, monkeypatch): monkeypatch.setenv("HOME", str(tmp_path)) runner = CliRunner() - result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "bash", "--install"]) assert result.exit_code == 0 - script_path = tmp_path / ".mvt-ios-complete.bash" + script_path = tmp_path / ".mvt-complete.bash" bashrc_path = tmp_path / ".bashrc" assert script_path.exists() - assert "_MVT_IOS_COMPLETE=bash_complete" in script_path.read_text( - encoding="utf-8" - ) + script = script_path.read_text(encoding="utf-8") + assert "_MVT_COMPLETE=bash_complete" in script + assert "_MVT_IOS_COMPLETE=bash_complete" in script + assert "_MVT_ANDROID_COMPLETE=bash_complete" in script bashrc = bashrc_path.read_text(encoding="utf-8") assert "[ -f" in bashrc - assert ".mvt-ios-complete.bash" in bashrc + assert ".mvt-complete.bash" in bashrc - result = runner.invoke(ios_cli, ["completion", "bash", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "bash", "--install"]) assert result.exit_code == 0 assert bashrc_path.read_text(encoding="utf-8") == bashrc @@ -65,14 +69,30 @@ class TestCompletionCommand: monkeypatch.setenv("HOME", str(tmp_path)) runner = CliRunner() - result = runner.invoke(android_cli, ["completion", "fish", "--install"]) + result = runner.invoke(mvt_cli, ["completion", "fish", "--install"]) assert result.exit_code == 0 - script_path = ( - tmp_path / ".config" / "fish" / "completions" / "mvt-android.fish" - ) + script_path = tmp_path / ".config" / "fish" / "conf.d" / "mvt-completion.fish" assert script_path.exists() - assert "_MVT_ANDROID_COMPLETE=fish_complete" in script_path.read_text( - encoding="utf-8" - ) + script = script_path.read_text(encoding="utf-8") + assert "_MVT_COMPLETE=fish_complete" in script + assert "_MVT_IOS_COMPLETE=fish_complete" in script + assert "_MVT_ANDROID_COMPLETE=fish_complete" in script assert not (tmp_path / ".fishrc").exists() + assert not (tmp_path / ".bashrc").exists() + assert not (tmp_path / ".zshrc").exists() + + def test_completion_install_without_shell_is_a_usage_error(self): + runner = CliRunner() + result = runner.invoke(mvt_cli, ["completion", "--install"]) + + assert result.exit_code == 2 + assert "A shell is required when using --install." in result.output + + def test_completion_is_not_a_command_of_the_platform_clis(self): + runner = CliRunner() + + assert "completion" not in ios_cli.commands + assert "completion" not in android_cli.commands + assert runner.invoke(ios_cli, ["completion"]).exit_code == 2 + assert runner.invoke(android_cli, ["completion"]).exit_code == 2 diff --git a/tests/test_custom_modules.py b/tests/test_custom_modules.py index a00faea..43ab621 100644 --- a/tests/test_custom_modules.py +++ b/tests/test_custom_modules.py @@ -1,3 +1,7 @@ +import hashlib +import importlib.metadata +import json + from click.testing import CliRunner from mvt.android.cli import check_bugreport @@ -5,7 +9,9 @@ from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF from mvt.android.cmd_check_backup import CmdAndroidCheckBackup from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport from mvt.android.cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs +from mvt.common import module_loader from mvt.common.module import MVTModule +from mvt.common.version import MVT_VERSION from mvt.ios.cli import check_backup, check_fs @@ -106,6 +112,150 @@ def test_custom_modules_load_from_environment_without_cli_flag(tmp_path, monkeyp assert "EnvBugreportModule" in result.output +class InstalledPackageModule(MVTModule): + supported_commands = (("ios", "check-backup"),) + + +def get_installed_package_modules(): + return [InstalledPackageModule] + + +def _fake_entry_points(monkeypatch, value, name="test-modules"): + entry_point = importlib.metadata.EntryPoint( + name=name, value=value, group=module_loader.MODULES_ENTRY_POINT_GROUP + ) + + def fake_entry_points(*, group): + assert group == module_loader.MODULES_ENTRY_POINT_GROUP + return [entry_point] + + monkeypatch.setattr( + module_loader.importlib.metadata, "entry_points", fake_entry_points + ) + + +def test_installed_module_package_loads_from_entry_point(monkeypatch): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + + modules = module_loader.load_custom_modules() + + assert modules == [InstalledPackageModule] + + +def test_broken_module_entry_point_is_skipped(monkeypatch, caplog): + _fake_entry_points(monkeypatch, "nonexistent_module_xyz:get_modules") + + with caplog.at_level("WARNING"): + modules = module_loader.load_custom_modules() + + assert modules == [] + assert "Unable to load modules from entry point" in caplog.text + + +def test_entry_point_module_deduplicated_against_paths(monkeypatch, tmp_path): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + module_path = _write_custom_module( + tmp_path / "custom.py", + "PathLoadedModule", + (("ios", "check-backup"),), + ) + + modules = module_loader.load_custom_modules([str(module_path)]) + + assert [module.__name__ for module in modules] == [ + "InstalledPackageModule", + "PathLoadedModule", + ] + + +def test_list_modules_shows_module_sources(tmp_path, caplog): + module_path = _write_custom_module( + tmp_path / "custom.py", + "SourcedBackupModule", + (("ios", "check-backup"),), + ) + file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest() + custom_modules = module_loader.load_custom_modules([str(module_path)]) + + from mvt.ios.cmd_check_backup import CmdIOSCheckBackup + + cmd = CmdIOSCheckBackup(target_path=str(tmp_path), custom_modules=custom_modules) + cmd.list_modules() + + assert f" - Modules from 'mvt@{MVT_VERSION}':" in caplog.text + assert ( + f" - Modules from '{module_path}' (sha256: {file_sha256}): SourcedBackupModule" + in caplog.text + ) + + +def test_builtin_module_origin(): + from mvt.ios.modules.backup import BACKUP_MODULES + + origin = module_loader.get_module_origin(BACKUP_MODULES[0]) + + assert origin.kind == "builtin" + assert origin.name == "mvt" + assert origin.version == MVT_VERSION + + +def test_installed_module_origin(monkeypatch): + _fake_entry_points(monkeypatch, f"{__name__}:get_installed_package_modules") + + modules = module_loader.load_custom_modules() + + origin = module_loader.get_module_origin(modules[0]) + assert origin.kind == "package" + assert origin.name == "test-modules" + + +def test_distribution_commit_read_from_direct_url(): + class FakeDistribution: + def read_text(self, filename): + assert filename == "direct_url.json" + return json.dumps( + { + "url": "https://github.com/example/example-modules", + "vcs_info": {"commit_id": "abc1234", "vcs": "git"}, + } + ) + + assert module_loader._distribution_commit(FakeDistribution()) == "abc1234" + + +def test_command_log_records_loaded_modules(tmp_path): + (tmp_path / "Manifest.db").touch() + (tmp_path / "Info.plist").touch() + module_path = _write_custom_module( + tmp_path / "custom.py", + "AuditedRunModule", + (("ios", "check-backup"),), + slug="audited_run_module", + ) + file_sha256 = hashlib.sha256(module_path.read_bytes()).hexdigest() + output_path = tmp_path / "out" + + result = CliRunner().invoke( + check_backup, + [ + "--module", + "AuditedRunModule", + "--load-module", + str(module_path), + "--output", + str(output_path), + str(tmp_path), + ], + ) + + assert result.exit_code == 0 + command_log = (output_path / "command.log").read_text(encoding="utf-8") + assert ( + f"Loaded 1 check-backup modules from '{module_path}' " + f"(sha256: {file_sha256}): AuditedRunModule" in command_log + ) + + class NestedBugreportModule(MVTModule): supported_commands = (("android", "check-bugreport"),)