mirror of
https://github.com/mvt-project/mvt.git
synced 2026-10-26 08:03:40 +01:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f8456059db | ||
|
|
efbab29f94 | ||
|
|
ecc22f54b7 | ||
|
|
90fc0ae351 | ||
|
|
dab82e38d0 | ||
|
|
4e031cd43f |
No files matched your search
@@ -37,8 +37,6 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v7
|
uses: actions/checkout@v7
|
||||||
with:
|
|
||||||
fetch-depth: 0 # the package version comes from the git tags
|
|
||||||
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
|
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
|
||||||
- name: Log in to the Container registry
|
- name: Log in to the Container registry
|
||||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
name: Release
|
|
||||||
|
|
||||||
on:
|
|
||||||
schedule:
|
|
||||||
- cron: "0 9 * * 1" # Mondays, 09:00 UTC
|
|
||||||
workflow_dispatch:
|
|
||||||
push:
|
|
||||||
tags: ["v*"]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
tag:
|
|
||||||
if: github.ref_type != 'tag'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
actions: write
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ github.token }}
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v7
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
- run: |
|
|
||||||
last=$(git describe --tags --abbrev=0 --match 'v*')
|
|
||||||
if [ "$(git rev-list "$last..HEAD" --count)" = 0 ]; then
|
|
||||||
echo "Nothing merged since $last"; exit 0
|
|
||||||
fi
|
|
||||||
tag=v$(date -u +%Y.%-m.%-d)
|
|
||||||
gh release create "$tag" --target "$GITHUB_SHA" --generate-notes
|
|
||||||
# Events made with GITHUB_TOKEN don't start other workflows; run them by hand.
|
|
||||||
gh workflow run release.yml --ref "$tag"
|
|
||||||
gh workflow run publish-release-docker.yml --ref "$tag"
|
|
||||||
|
|
||||||
publish:
|
|
||||||
if: github.ref_type == 'tag'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
id-token: write # PyPI trusted publishing
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v7
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
- uses: astral-sh/setup-uv@v10.0.0
|
|
||||||
- run: uv build && uv publish
|
|
||||||
@@ -5,10 +5,6 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches: [ main ]
|
branches: [ main ]
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
pull-requests: write # coverage comment
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
name: Run Python Tests
|
name: Run Python Tests
|
||||||
@@ -17,10 +13,6 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
||||||
env:
|
|
||||||
# Takes precedence over .python-version, which otherwise makes `uv run`
|
|
||||||
# rebuild the venv with 3.10 and test every matrix entry on 3.10.
|
|
||||||
UV_PYTHON: ${{ matrix.python-version }}
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v7
|
- uses: actions/checkout@v7
|
||||||
@@ -34,23 +26,16 @@ jobs:
|
|||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
- name: Install Python dependencies
|
- name: Install Python dependencies
|
||||||
run: |
|
run: |
|
||||||
uv sync --locked --group dev
|
uv sync --locked --group dev --python ${{ matrix.python-version }}
|
||||||
- name: Test with pytest
|
- name: Test with pytest
|
||||||
run: |
|
run: |
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
make test-ci | tee pytest-coverage.txt
|
make test-ci | tee pytest-coverage.txt
|
||||||
- name: Coverage job summary
|
|
||||||
run: uv run coverage report --format=markdown --show-missing --skip-covered >> "$GITHUB_STEP_SUMMARY"
|
|
||||||
|
|
||||||
- name: Pytest coverage comment
|
- name: Pytest coverage comment
|
||||||
# One comment per PR, not one per matrix entry. PRs from forks get a
|
continue-on-error: true # Workflows running on a fork can't post comments
|
||||||
# read-only token and can't post; the job summary above still works.
|
uses: MishaKav/pytest-coverage-comment@main
|
||||||
if: github.event_name == 'pull_request' && matrix.python-version == '3.13'
|
if: github.event_name == 'pull_request'
|
||||||
continue-on-error: true
|
|
||||||
uses: MishaKav/pytest-coverage-comment@v1.12.2
|
|
||||||
with:
|
with:
|
||||||
pytest-coverage-path: ./pytest-coverage.txt
|
pytest-coverage-path: ./pytest-coverage.txt
|
||||||
junitxml-path: ./pytest.xml
|
junitxml-path: ./pytest.xml
|
||||||
# The full table with per-line links exceeds GitHub's 65536-char comment limit.
|
|
||||||
report-only-changed-files: true
|
|
||||||
remove-links-to-lines: true
|
|
||||||
@@ -122,7 +122,6 @@ RUN apt-get update \
|
|||||||
binutils \
|
binutils \
|
||||||
default-jre-headless \
|
default-jre-headless \
|
||||||
file \
|
file \
|
||||||
git \
|
|
||||||
jq \
|
jq \
|
||||||
less \
|
less \
|
||||||
libcurl4 \
|
libcurl4 \
|
||||||
|
|||||||
@@ -32,3 +32,8 @@ clean:
|
|||||||
dist:
|
dist:
|
||||||
$(UV) build
|
$(UV) build
|
||||||
|
|
||||||
|
upload:
|
||||||
|
$(UV) tool run twine upload dist/*
|
||||||
|
|
||||||
|
test-upload:
|
||||||
|
$(UV) tool run twine upload --repository testpypi dist/*
|
||||||
@@ -58,33 +58,34 @@ For alternative installation options and known issues, please refer to the [docu
|
|||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
MVT provides three commands: `mvt-ios` and `mvt-android` analyse acquisitions from devices of that platform, and `mvt` hosts what belongs to neither: `version`, `completion`, `plugins` and `download-iocs` (`version` and `download-iocs` remain available on the platform commands for now). Running `mvt` on its own shows the installed version, update notices and the available commands. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
|
MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
|
||||||
|
|
||||||
Pass `--verbose` to any of the three commands, before the command name (`mvt-ios --verbose check-backup ...`), for debug output. The `--verbose` option the `check-*` commands accept after their name still works but is kept for compatibility only and will be removed in a future release.
|
|
||||||
|
|
||||||
### Shell completion
|
### Shell completion
|
||||||
|
|
||||||
MVT can generate a shell completion script for Bash, Zsh, and Fish which covers `mvt`, `mvt-ios` and `mvt-android`:
|
MVT can generate shell completion scripts for Bash, Zsh, and Fish:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mvt completion
|
mvt-ios completion
|
||||||
|
mvt-android completion
|
||||||
```
|
```
|
||||||
|
|
||||||
The command prints setup instructions by default. To generate the completion script directly, pass the shell name:
|
The commands print setup instructions by default. To generate a completion script directly, pass the shell name:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mvt completion bash
|
mvt-ios completion bash
|
||||||
|
mvt-android completion zsh
|
||||||
```
|
```
|
||||||
|
|
||||||
MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details.
|
MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details.
|
||||||
|
Module-running `check-*` commands can load custom Python modules with
|
||||||
Plugin packages extend MVT with additional forensic modules, which run inside
|
`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the
|
||||||
the `check-*` commands, and with top-level commands on `mvt`, `mvt-ios` and
|
|
||||||
`mvt-android`. See the
|
|
||||||
[development documentation](https://docs.mvt.re/en/latest/development/) for
|
[development documentation](https://docs.mvt.re/en/latest/development/) for
|
||||||
writing and installing them, and the
|
details.
|
||||||
|
|
||||||
|
Users can also add top-level commands to `mvt-ios` and `mvt-android` from
|
||||||
|
installed Python packages or local files and folders. See the
|
||||||
[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/)
|
[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/)
|
||||||
for the entry points a package registers commands in.
|
for the plugin entry points and `--load-command` interface.
|
||||||
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ mvt-android check-intrusion-logs --output /path/to/results/ /path/to/intrusion-l
|
|||||||
| `-l, --list-modules` | List the available intrusion-log modules and exit. |
|
| `-l, --list-modules` | List the available intrusion-log modules and exit. |
|
||||||
| `-m, --module NAME` | Run a single module (e.g. `DnsEvent`) instead of all of them. |
|
| `-m, --module NAME` | Run a single module (e.g. `DnsEvent`) instead of all of them. |
|
||||||
| `-t, --timezone TZ` | IANA timezone name for the device (e.g. `Europe/Paris`). When set, event timestamps are converted to the device's local time instead of UTC. |
|
| `-t, --timezone TZ` | IANA timezone name for the device (e.g. `Europe/Paris`). When set, event timestamps are converted to the device's local time instead of UTC. |
|
||||||
| `-v, --verbose` | Verbose logging. Kept for compatibility and to be removed in a future release: pass `--verbose` to `mvt-android` itself instead. |
|
| `-v, --verbose` | Verbose logging. |
|
||||||
|
|
||||||
## Modules
|
## Modules
|
||||||
|
|
||||||
|
|||||||
+22
-22
@@ -6,61 +6,61 @@ Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fi
|
|||||||
|
|
||||||
To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using.
|
To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using.
|
||||||
|
|
||||||
`mvt completion` generates one script which covers `mvt`, `mvt-ios` and `mvt-android`. The following describes how to generate that script and add it to your shell configuration.
|
The following describes how to generate the command completion scripts and add them to your shell configuration.
|
||||||
|
|
||||||
> **Note: You will need to start a new shell for the changes to take effect.**
|
> **Note: You will need to start a new shell for the changes to take effect.**
|
||||||
|
|
||||||
### For Bash
|
### For Bash
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Generate the bash completion script
|
# Generate bash completion scripts
|
||||||
mvt completion bash > ~/.mvt-complete.bash
|
mvt-ios completion bash > ~/.mvt-ios-complete.bash
|
||||||
|
mvt-android completion bash > ~/.mvt-android-complete.bash
|
||||||
```
|
```
|
||||||
|
|
||||||
Add the following to `~/.bashrc`:
|
Add the following to `~/.bashrc`:
|
||||||
```bash
|
```bash
|
||||||
# source the mvt completion script
|
# source mvt completion scripts
|
||||||
[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash
|
[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash
|
||||||
|
[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash
|
||||||
```
|
```
|
||||||
|
|
||||||
### For Zsh
|
### For Zsh
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Generate the zsh completion script
|
# Generate zsh completion scripts
|
||||||
mvt completion zsh > ~/.mvt-complete.zsh
|
mvt-ios completion zsh > ~/.mvt-ios-complete.zsh
|
||||||
|
mvt-android completion zsh > ~/.mvt-android-complete.zsh
|
||||||
```
|
```
|
||||||
|
|
||||||
Add the following to `~/.zshrc`:
|
Add the following to `~/.zshrc`:
|
||||||
```bash
|
```bash
|
||||||
# source the mvt completion script
|
# source mvt completion scripts
|
||||||
[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh
|
[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh
|
||||||
|
[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh
|
||||||
```
|
```
|
||||||
|
|
||||||
### For Fish
|
### For Fish
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Generate the fish completion script
|
# Generate fish completion scripts
|
||||||
mkdir -p ~/.config/fish/conf.d
|
mkdir -p ~/.config/fish/completions
|
||||||
mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish
|
mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish
|
||||||
|
mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish
|
||||||
```
|
```
|
||||||
|
|
||||||
Fish loads the files in `~/.config/fish/conf.d` automatically.
|
Fish loads completion files from `~/.config/fish/completions` automatically.
|
||||||
|
|
||||||
### Automatic Installation
|
### Automatic Installation
|
||||||
|
|
||||||
MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`:
|
MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mvt completion bash --install
|
mvt-ios completion bash --install
|
||||||
|
mvt-android completion bash --install
|
||||||
```
|
```
|
||||||
|
|
||||||
Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/conf.d` and changes no shell configuration.
|
Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`.
|
||||||
|
|
||||||
!!! note
|
|
||||||
|
|
||||||
Earlier versions generated one script per command, with `mvt-ios completion`
|
|
||||||
and `mvt-android completion`. Files written by them keep working. When you
|
|
||||||
switch to the single script, remove the old files and the lines which load
|
|
||||||
them from your shell configuration.
|
|
||||||
|
|
||||||
For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion).
|
For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion).
|
||||||
|
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
# Custom CLI Commands
|
# Custom CLI Commands
|
||||||
|
|
||||||
MVT can load additional top-level commands into `mvt`, `mvt-ios` and
|
MVT can load additional top-level commands into `mvt-ios` and `mvt-android`.
|
||||||
`mvt-android`. A command package chooses which of the three each of its
|
|
||||||
commands is added to.
|
|
||||||
Custom commands are different from [custom forensic modules](index.md#custom-modules):
|
Custom commands are different from [custom forensic modules](index.md#custom-modules):
|
||||||
commands add new CLI operations, while modules add analysis steps to existing
|
commands add new CLI operations, while modules add analysis steps to existing
|
||||||
`check-*` commands.
|
`check-*` commands.
|
||||||
@@ -15,8 +13,8 @@ commands add new CLI operations, while modules add analysis steps to existing
|
|||||||
|
|
||||||
## Install a Command Package
|
## Install a Command Package
|
||||||
|
|
||||||
Python packages can register a Click command or group on one or more of the MVT CLIs.
|
Python packages can register a Click command or group for either MVT CLI. A
|
||||||
A minimal package can expose this command from `my_mvt_plugin.py`:
|
minimal package can expose this command from `my_mvt_plugin.py`:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
import click
|
import click
|
||||||
@@ -29,10 +27,6 @@ def summarize(path):
|
|||||||
click.echo(f"Summarizing {path}")
|
click.echo(f"Summarizing {path}")
|
||||||
```
|
```
|
||||||
|
|
||||||
Log through `get_plugin_logger(__name__)` from `mvt.plugin`. Records logged
|
|
||||||
through `logging.getLogger(__name__)` do not reach `command.log`, and MVT's
|
|
||||||
console handler does not show them.
|
|
||||||
|
|
||||||
Register the object in the package's `pyproject.toml`. The entry-point name is
|
Register the object in the package's `pyproject.toml`. The entry-point name is
|
||||||
the command users invoke:
|
the command users invoke:
|
||||||
|
|
||||||
@@ -44,13 +38,9 @@ summarize = "my_mvt_plugin:summarize"
|
|||||||
summarize = "my_mvt_plugin:summarize"
|
summarize = "my_mvt_plugin:summarize"
|
||||||
```
|
```
|
||||||
|
|
||||||
Each entry-point group adds the command to one CLI: `mvt.ios.cli_plugins` to
|
Use only the iOS or Android group if the command is platform-specific. After
|
||||||
`mvt-ios`, `mvt.android.cli_plugins` to `mvt-android` and `mvt.cli_plugins` to
|
installing the package in the same environment as MVT, it appears directly in
|
||||||
`mvt`. Register the command in the group of every CLI which should offer it: a
|
the appropriate CLI:
|
||||||
platform-specific command belongs in one platform group, and a command which
|
|
||||||
handles acquisitions of both platforms, as above, in both. After installing the
|
|
||||||
package in the same environment as MVT, the command appears directly in those
|
|
||||||
CLIs:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mvt-ios summarize ./ios-backup
|
mvt-ios summarize ./ios-backup
|
||||||
@@ -64,35 +54,16 @@ pipx inject mvt my-mvt-plugin
|
|||||||
```
|
```
|
||||||
|
|
||||||
When MVT is installed in an active virtual environment, install the plugin with
|
When MVT is installed in an active virtual environment, install the plugin with
|
||||||
`pip` in that environment. `mvt plugins list` shows the installed packages and
|
`pip` in that environment.
|
||||||
the commands they add, see [Managing Plugins](plugins.md).
|
|
||||||
|
|
||||||
Command packages that need their own settings, such as an API key, should store
|
Command packages that need their own settings, such as an API key, should store
|
||||||
them in a namespaced [plugin configuration file](plugin_configuration.md)
|
them in a namespaced [plugin configuration file](plugin_configuration.md)
|
||||||
rather than in MVT's own `config.yaml`.
|
rather than in MVT's own `config.yaml`.
|
||||||
|
|
||||||
### Commands on `mvt`
|
## Load a Command File
|
||||||
|
|
||||||
A MVT plugin command can also add sub-commands to the base `mvt` command. This can be used for commands which are not tied to a particular forensic platform:
|
For local commands that are not packaged, create a Python file that exports one
|
||||||
|
Click command or group named `cli`:
|
||||||
```toml
|
|
||||||
[project.entry-points."mvt.cli_plugins"]
|
|
||||||
my-plugin = "my_mvt_plugin:my_plugin"
|
|
||||||
```
|
|
||||||
|
|
||||||
Commands in this group are added to `mvt` only, so this one is invoked as
|
|
||||||
`mvt my-plugin`. A command on `mvt` has nothing but its name to say which
|
|
||||||
plugin it belongs to, so name it after the plugin, and make it a Click group
|
|
||||||
when the plugin has several operations to offer, such as
|
|
||||||
`mvt my-plugin configure`.
|
|
||||||
|
|
||||||
## Developing a Command Locally
|
|
||||||
|
|
||||||
A package is how a command is distributed. While a command is being written,
|
|
||||||
MVT can load it straight from its file instead, so the package need not be
|
|
||||||
reinstalled after every change; an editable install of the package does the
|
|
||||||
same through its entry points. Create a Python file that exports one Click
|
|
||||||
command or group named `cli`:
|
|
||||||
|
|
||||||
```python
|
```python
|
||||||
import click
|
import click
|
||||||
@@ -115,82 +86,20 @@ mvt-ios --load-command ./case_summary.py case-summary ./ios-backup
|
|||||||
non-hidden top-level `*.py` files in sorted order and skips `__init__.py`.
|
non-hidden top-level `*.py` files in sorted order and skips `__init__.py`.
|
||||||
Every loaded file must export one `cli` object.
|
Every loaded file must export one `cli` object.
|
||||||
|
|
||||||
To load a file or folder on every invocation, set the environment variable of
|
To load a file or folder on every invocation, set the platform-specific
|
||||||
the CLI the commands belong on. Like the entry-point groups, each variable adds
|
environment variable:
|
||||||
its commands to one CLI only:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
export MVT_CUSTOM_COMMANDS=./commands
|
|
||||||
export MVT_IOS_CUSTOM_COMMANDS=./ios_commands
|
export MVT_IOS_CUSTOM_COMMANDS=./ios_commands
|
||||||
export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands
|
export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands
|
||||||
```
|
```
|
||||||
|
|
||||||
## Building a Module-Running Command
|
|
||||||
|
|
||||||
A command which runs forensic modules over an acquisition subclasses `Command`.
|
|
||||||
`Command` creates the output folder and writes `command.log`. It orders the
|
|
||||||
modules, resolves their dependencies and runs them. It writes the result files,
|
|
||||||
`alerts.json` and `info.json`. The subclass sets `platform`, `name` and
|
|
||||||
`modules`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
from mvt.plugin import Command, MVTModule, convert_unix_to_iso, get_plugin_logger
|
|
||||||
|
|
||||||
log = get_plugin_logger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class APKManifest(MVTModule):
|
|
||||||
supported_commands = (("android", "check-apks"),)
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
self.results = [{"checked_at": convert_unix_to_iso(0)}]
|
|
||||||
|
|
||||||
|
|
||||||
class CmdCheckAPKs(Command):
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
self.platform = "android"
|
|
||||||
self.name = "check-apks"
|
|
||||||
self.modules = [APKManifest]
|
|
||||||
```
|
|
||||||
|
|
||||||
`platform` and `name` are the pair the modules declare in
|
|
||||||
`supported_commands`. `modules` lists the module classes the command runs,
|
|
||||||
imported directly. A command which runs modules of another plugin depends on
|
|
||||||
that package in `pyproject.toml` and imports them the same way. `init()`,
|
|
||||||
`module_init(module)` and `finish()` are optional hooks. `run()` calls them
|
|
||||||
before the run, before each module and after the run.
|
|
||||||
|
|
||||||
Wrap the command in a Click command with an `--output` option. Run it, then
|
|
||||||
print the alert summary:
|
|
||||||
|
|
||||||
```python
|
|
||||||
import click
|
|
||||||
|
|
||||||
|
|
||||||
@click.command("check-apks")
|
|
||||||
@click.option("--output", "-o", type=click.Path(exists=False))
|
|
||||||
@click.argument("TARGET_PATH", type=click.Path(exists=True))
|
|
||||||
def cli(output, target_path):
|
|
||||||
cmd = CmdCheckAPKs(target_path=target_path, results_path=output)
|
|
||||||
log.info("Checking APK files at path: %s", target_path)
|
|
||||||
cmd.run()
|
|
||||||
cmd.show_alerts_brief()
|
|
||||||
```
|
|
||||||
|
|
||||||
The `--verbose` option of `mvt`, `mvt-ios` and `mvt-android` applies to the
|
|
||||||
command. The command defines no `--verbose` option of its own. The pair a
|
|
||||||
plugin command adds is not listed anywhere in MVT. Name it in the plugin's
|
|
||||||
README.
|
|
||||||
|
|
||||||
## Naming and Errors
|
## Naming and Errors
|
||||||
|
|
||||||
Built-in MVT commands cannot be replaced. External command names must also be
|
Built-in MVT commands cannot be replaced. External command names must also be
|
||||||
unique on each CLI; when installed packages or environment paths collide, MVT
|
unique; when installed packages or environment paths collide, MVT keeps the
|
||||||
keeps the first command and logs a warning. The environment path of a CLI is
|
first command and logs a warning. A collision from an explicit
|
||||||
registered before its installed packages, so a command loaded from there wins a
|
`--load-command` is a usage error.
|
||||||
collision with a package. A collision from an explicit `--load-command` is a
|
|
||||||
usage error.
|
|
||||||
|
|
||||||
A package entry point or environment command that cannot be imported appears
|
A package entry point or environment command that cannot be imported appears
|
||||||
as a marked broken command without preventing other MVT commands from working.
|
as a marked broken command without preventing other MVT commands from working.
|
||||||
|
|||||||
+56
-218
@@ -48,21 +48,51 @@ configuration problem: the command logs a warning and runs no modules at all.
|
|||||||
|
|
||||||
## Custom modules
|
## Custom modules
|
||||||
|
|
||||||
MVT's module-running `check-*` commands can run forensic modules which are not
|
Module-running `check-*` commands can load custom modules from Python files that
|
||||||
part of MVT. Custom modules are distributed as plugin packages. A Python
|
are not installed as part of MVT. Load one file with:
|
||||||
package installed next to MVT registers its modules through an entry point.
|
|
||||||
The modules then load automatically in every command they support, see
|
|
||||||
[Installed module packages](#installed-module-packages). `mvt plugins list`
|
|
||||||
shows the installed packages and where each one was installed from.
|
|
||||||
|
|
||||||
MVT can also load module files by path, with `--load-module` and
|
```bash
|
||||||
`MVT_CUSTOM_MODULES`, see
|
mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup
|
||||||
[Developing modules locally](#developing-modules-locally). This can be used
|
```
|
||||||
while writing a module. Use a Python package to distribute one.
|
|
||||||
|
|
||||||
A custom module declares in `supported_commands` the platform and command
|
You can also load a folder. MVT loads non-hidden top-level `*.py` files in
|
||||||
pairs it runs in. A module with empty `supported_commands` does not run and
|
sorted order and skips `__init__.py`:
|
||||||
MVT logs a warning. The nine pairs are:
|
|
||||||
|
```bash
|
||||||
|
mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This
|
||||||
|
folder is loaded before any `--load-module` path:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip
|
||||||
|
```
|
||||||
|
|
||||||
|
Custom modules are normal `MVTModule` subclasses:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from mvt.common.module import MVTModule
|
||||||
|
|
||||||
|
|
||||||
|
class ExampleCustomModule(MVTModule):
|
||||||
|
supported_commands = (("ios", "check-backup"), ("ios", "check-fs"))
|
||||||
|
slug = "example_custom_module"
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.results = [{"message": "custom module ran"}]
|
||||||
|
|
||||||
|
def check_indicators(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def serialize(self, result):
|
||||||
|
return None
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `supported_commands` to declare the platform/command pairs a module
|
||||||
|
supports. Empty `supported_commands` means the module will not run and MVT logs
|
||||||
|
a warning. This explicit declaration is required for every command. Supported
|
||||||
|
pairs are:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
("ios", "check-backup")
|
("ios", "check-backup")
|
||||||
@@ -76,88 +106,13 @@ MVT logs a warning. The nine pairs are:
|
|||||||
("android", "check-iocs")
|
("android", "check-iocs")
|
||||||
```
|
```
|
||||||
|
|
||||||
`check-iocs` re-checks stored results rather than an acquisition. It matches
|
Custom modules can depend on existing MVT module classes. Dependencies are
|
||||||
every `<slug>.json` file in the results folder to the module with that slug.
|
resolved with the same ordering logic as built-in modules, and custom modules
|
||||||
It then runs that module's `check_indicators()` again. A module which
|
are appended after built-ins before ordering:
|
||||||
implements `check_indicators()` is included in `check-iocs` for its platform.
|
|
||||||
It does not need to declare the `check-iocs` pair.
|
|
||||||
|
|
||||||
### Writing a module
|
|
||||||
|
|
||||||
A module subclasses `MVTModule` or one of the base classes below and
|
|
||||||
implements `run()`. `check_indicators()` and `serialize()` are optional. The
|
|
||||||
first matches results against IOCs or detections. The second returns timeline
|
|
||||||
records.
|
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from mvt.plugin import IOSExtraction, convert_unix_to_iso
|
from mvt.common.module import MVTModule
|
||||||
|
from mvt.ios.modules.backup.manifest import Manifest
|
||||||
|
|
||||||
class ExampleCustomModule(IOSExtraction):
|
|
||||||
supported_commands = (
|
|
||||||
("ios", "check-backup"),
|
|
||||||
("ios", "check-fs"),
|
|
||||||
)
|
|
||||||
slug = "example_custom_module"
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
self.results = [{"checked_at": convert_unix_to_iso(0)}]
|
|
||||||
|
|
||||||
def check_indicators(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def serialize(self, result):
|
|
||||||
return None
|
|
||||||
```
|
|
||||||
|
|
||||||
The base classes are:
|
|
||||||
|
|
||||||
- `MVTModule`: the base of every module. It provides `self.results`,
|
|
||||||
`self.alertstore`, `self.log`, `self.indicators` and
|
|
||||||
`get_dependency_results()`. Subclass it directly for a module which reads
|
|
||||||
only the results of other modules.
|
|
||||||
- `IOSExtraction`: `("ios", "check-backup")` and `("ios", "check-fs")`. Adds
|
|
||||||
`_find_ios_database()`, which locates a module's database in a backup or in
|
|
||||||
a filesystem dump and repairs it if it is malformed. Adds
|
|
||||||
`_get_backup_files_from_manifest()`, `_get_backup_file_from_id()` and
|
|
||||||
`_get_fs_files_from_patterns()`. Adds `_open_sqlite_db()`, which opens a
|
|
||||||
database read-only.
|
|
||||||
- `SysdiagnoseExtraction`: `("ios", "check-sysdiagnose")`. MVT extracts the
|
|
||||||
archive and calls `from_sysdiagnose_folder()` before `run()`. The module
|
|
||||||
reads files with `_get_files_by_pattern()` and `_get_file_content()`.
|
|
||||||
`ips_files` lists the crash reports. See
|
|
||||||
[Check an iOS Sysdiagnose](../ios/sysdiagnose.md).
|
|
||||||
- `AndroidQFModule`: `("android", "check-androidqf")`. MVT calls `from_dir()`
|
|
||||||
or `from_zip()` with the file list of the acquisition. The module reads
|
|
||||||
files with `_get_files_by_pattern()` and `_get_file_content()`.
|
|
||||||
`_get_device_timezone()` returns the device timezone.
|
|
||||||
- `AndroidBackupModule`: `("android", "check-backup")`. MVT calls `from_dir()`
|
|
||||||
or `from_ab()`. The module reads files with `_get_files_by_pattern()` and
|
|
||||||
`_get_file_content()`.
|
|
||||||
- `BugReportModule`: `("android", "check-bugreport")`. MVT calls `from_dir()`
|
|
||||||
or `from_zip()`. The module reads files with `_get_files_by_pattern()`,
|
|
||||||
`_get_files_by_patterns()` and `_get_file_content()`.
|
|
||||||
`_get_dumpstate_file()` returns the dumpstate file, and
|
|
||||||
`_get_file_modification_time()` the modification time of a file.
|
|
||||||
|
|
||||||
The underscore-named helpers are internal to the base classes. Plugin modules
|
|
||||||
can call them. Their names and signatures can change between releases. Read
|
|
||||||
the base class in `src/mvt/ios/modules` or `src/mvt/android/modules` before
|
|
||||||
relying on one.
|
|
||||||
|
|
||||||
### Depending on a built-in module
|
|
||||||
|
|
||||||
A module which post-processes records generated by one or more built-in MVT
|
|
||||||
modules must declare the source modules in `dependencies`. It reads their
|
|
||||||
results with `get_dependency_results()`. Import the class from its family
|
|
||||||
package: `mvt.ios.modules.backup`, `mvt.ios.modules.fs`,
|
|
||||||
`mvt.ios.modules.mixed`, `mvt.android.modules.androidqf`,
|
|
||||||
`mvt.android.modules.backup`, `mvt.android.modules.bugreport` or
|
|
||||||
`mvt.android.modules.intrusion_logs`.
|
|
||||||
|
|
||||||
```python
|
|
||||||
from mvt.ios.modules.backup import Manifest
|
|
||||||
from mvt.plugin import MVTModule
|
|
||||||
|
|
||||||
|
|
||||||
class DependentCustomModule(MVTModule):
|
class DependentCustomModule(MVTModule):
|
||||||
@@ -169,85 +124,6 @@ class DependentCustomModule(MVTModule):
|
|||||||
self.results = [{"manifest_entries": len(manifest_results)}]
|
self.results = [{"manifest_entries": len(manifest_results)}]
|
||||||
```
|
```
|
||||||
|
|
||||||
Dependencies are ordered as for the built-in modules, with custom modules
|
|
||||||
appended after the built-ins, see [Module dependencies](#module-dependencies).
|
|
||||||
A dependency has to run in every command the module supports. Where it does
|
|
||||||
not, MVT skips the module with a warning.
|
|
||||||
|
|
||||||
`get_dependency_results()` returns the plain dictionaries the module produced.
|
|
||||||
They are the same records it writes to `<slug>.json`. Typed results per module
|
|
||||||
are planned.
|
|
||||||
|
|
||||||
### Replacing a built-in module
|
|
||||||
|
|
||||||
A custom module which extends a built-in one runs alongside it, and when the
|
|
||||||
two share a slug they write to the same results file. MVT warns about that
|
|
||||||
collision, naming both modules, where each came from and the file the later one
|
|
||||||
overwrites, but it runs them both. Set `replaces` to the class the module
|
|
||||||
supersedes to take that module's place instead: when both are available to a
|
|
||||||
command, the named module is dropped from the run.
|
|
||||||
|
|
||||||
```python
|
|
||||||
from mvt.ios.modules.backup import Manifest as BuiltinManifest
|
|
||||||
|
|
||||||
|
|
||||||
class Manifest(BuiltinManifest):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = BuiltinManifest
|
|
||||||
```
|
|
||||||
|
|
||||||
Keeping the class name of the replaced module, as above, keeps the name
|
|
||||||
`--module` selects the module by and the slug its results file is named after.
|
|
||||||
A replacement with a different class name writes to a file named after its own
|
|
||||||
slug, unless it sets `slug` to the slug of the module it replaces, and
|
|
||||||
`--module` still selects it by the name of the replaced module, which MVT logs.
|
|
||||||
Taking over the slug of a replaced module is not a collision and is not warned
|
|
||||||
about, because that module is no longer part of the run.
|
|
||||||
|
|
||||||
Modules which depend on the replaced module receive the replacement instead, so
|
|
||||||
`get_dependency_results(BuiltinManifest)` returns the replacement's results. A
|
|
||||||
module must not depend on the module it replaces: that module does not run, so
|
|
||||||
the replacement has to produce the data itself, and MVT logs a warning when a
|
|
||||||
module declares both.
|
|
||||||
|
|
||||||
A replacement takes on the obligations of the module it replaces. If it
|
|
||||||
declares a dependency the command does not provide, it is skipped like any
|
|
||||||
other module with an unavailable dependency, and the module it replaced stays
|
|
||||||
out of the run: neither of them produces results, and the modules depending on
|
|
||||||
the replaced module are skipped as well.
|
|
||||||
|
|
||||||
Subclassing the replaced module is not required, but a replacement which is not
|
|
||||||
a subclass is logged with a warning, because its results may not be what the
|
|
||||||
modules depending on the replaced module expect. Naming a module the command
|
|
||||||
does not run has no effect, and modules which replace each other in a cycle all
|
|
||||||
keep running and replace nothing. Every applied substitution is logged, so it
|
|
||||||
is recorded in `command.log` when the command runs with an `--output` folder.
|
|
||||||
|
|
||||||
Every command resolves replacements on its own. `check-iocs` matches stored
|
|
||||||
results files against the slugs of the modules available for that command. A
|
|
||||||
replacement which subclasses the module it replaces inherits its
|
|
||||||
`check_indicators()`. It is then part of `check-iocs` for its platform and
|
|
||||||
re-checks the results file named after its slug. A replacement with no
|
|
||||||
`check_indicators()` is not part of `check-iocs`. The built-in module it
|
|
||||||
replaced re-checks the file. `check-iocs` matches `--module` on the class name
|
|
||||||
only. Pass a differently named replacement's own name there, not the name of
|
|
||||||
the module it replaces.
|
|
||||||
|
|
||||||
### Importing from MVT
|
|
||||||
|
|
||||||
Import from `mvt.plugin` if it has what you need. The names it exports are
|
|
||||||
kept working on a best-effort basis. A change to one of them is announced in
|
|
||||||
the release notes. Anything else in `mvt` can be imported too, but may change
|
|
||||||
between releases without notice. The plugin interface is best effort.
|
|
||||||
|
|
||||||
`mvt.plugin` exports the base classes above and `Command`, `Alert` and
|
|
||||||
`AlertLevel`, the result types, `DatabaseNotFoundError` and
|
|
||||||
`DatabaseCorruptedError`, the timestamp converters, the settings API of
|
|
||||||
[Plugin Configuration](plugin_configuration.md), MVT's own `settings`,
|
|
||||||
`get_plugin_logger()` and `MVT_VERSION`. `src/mvt/plugin.py` holds the list.
|
|
||||||
Read MVT's `settings` for values such as `NETWORK_ACCESS_ALLOWED` and
|
|
||||||
`NETWORK_TIMEOUT`. Plugin values go in the plugin's own settings file.
|
|
||||||
|
|
||||||
## Installed module packages
|
## Installed module packages
|
||||||
|
|
||||||
Python packages can register modules so they load automatically in every
|
Python packages can register modules so they load automatically in every
|
||||||
@@ -257,14 +133,14 @@ the package's `pyproject.toml`:
|
|||||||
|
|
||||||
```toml
|
```toml
|
||||||
[project.entry-points."mvt.modules"]
|
[project.entry-points."mvt.modules"]
|
||||||
mvt-plugin-example-org = "mvt_plugin_example_org:get_modules"
|
mvt-plugin-amnesty-custom = "mvt_plugin_amnesty_custom:get_modules"
|
||||||
```
|
```
|
||||||
|
|
||||||
The entry point must resolve to an iterable of `MVTModule` subclasses, or to
|
The entry point must resolve to an iterable of `MVTModule` subclasses, or to
|
||||||
a callable returning one:
|
a callable returning one:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from mvt.plugin import MVTModule
|
from mvt.common.module import MVTModule
|
||||||
|
|
||||||
|
|
||||||
class PackagedModule(MVTModule):
|
class PackagedModule(MVTModule):
|
||||||
@@ -278,10 +154,6 @@ def get_modules() -> list[type[MVTModule]]:
|
|||||||
return [PackagedModule]
|
return [PackagedModule]
|
||||||
```
|
```
|
||||||
|
|
||||||
`get_modules()` is the package's module list, written by hand. A package which
|
|
||||||
keeps its modules in separate files imports each class there and lists it. A
|
|
||||||
module missing from the list does not load.
|
|
||||||
|
|
||||||
Installed modules follow the same rules as other custom modules: each module
|
Installed modules follow the same rules as other custom modules: each module
|
||||||
must declare `supported_commands`, and dependencies are resolved with the
|
must declare `supported_commands`, and dependencies are resolved with the
|
||||||
standard ordering logic. A broken entry point is skipped with a warning and
|
standard ordering logic. A broken entry point is skipped with a warning and
|
||||||
@@ -292,7 +164,7 @@ from sources you trust.
|
|||||||
For a `pipx` installation of MVT, inject the package into MVT's environment:
|
For a `pipx` installation of MVT, inject the package into MVT's environment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pipx inject mvt mvt-plugin-example-org
|
pipx inject mvt mvt-plugin-amnesty-custom
|
||||||
```
|
```
|
||||||
|
|
||||||
Module packages that need their own settings, such as an API key, should store
|
Module packages that need their own settings, such as an API key, should store
|
||||||
@@ -303,9 +175,9 @@ rather than in MVT's own `config.yaml`.
|
|||||||
|
|
||||||
Name module packages `mvt-plugin-<name>` (import package `mvt_plugin_<name>`),
|
Name module packages `mvt-plugin-<name>` (import package `mvt_plugin_<name>`),
|
||||||
and include the name of the publishing organization or author so packages from
|
and include the name of the publishing organization or author so packages from
|
||||||
different groups do not collide: for example, an organisation's custom modules
|
different groups do not collide: for example, Amnesty International's custom
|
||||||
would be distributed as `mvt-plugin-example-org` with the import package
|
modules would be distributed as `mvt-plugin-amnesty-custom` with the import
|
||||||
`mvt_plugin_example_org`.
|
package `mvt_plugin_amnesty_custom`.
|
||||||
|
|
||||||
The prefix makes module packages easy to find on PyPI and keeps their import
|
The prefix makes module packages easy to find on PyPI and keeps their import
|
||||||
names from clashing with unrelated Python packages. It is a convention, not a
|
names from clashing with unrelated Python packages. It is a convention, not a
|
||||||
@@ -324,42 +196,11 @@ came from. MVT's own modules log under their dotted path (for example
|
|||||||
MVT's internal logger tree:
|
MVT's internal logger tree:
|
||||||
|
|
||||||
- Installed packages log under `mvt.ext.<package>`, with the `mvt_plugin_`
|
- Installed packages log under `mvt.ext.<package>`, with the `mvt_plugin_`
|
||||||
prefix stripped: modules in `mvt_plugin_example_org` log as
|
prefix stripped: modules in `mvt_plugin_amnesty_custom` log as
|
||||||
`mvt.ext.example_org.*`.
|
`mvt.ext.amnesty_custom.*`.
|
||||||
- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as
|
- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as
|
||||||
`mvt.ext.<file name>`.
|
`mvt.ext.<file name>`.
|
||||||
|
|
||||||
Outside a module class, for example in a custom command line handler, log
|
|
||||||
through `get_plugin_logger(__name__)`. It returns a logger in the same
|
|
||||||
namespace.
|
|
||||||
|
|
||||||
## Developing modules locally
|
|
||||||
|
|
||||||
While a module is being written, load it from its file. `--load-module` takes a
|
|
||||||
Python file, or a folder of them, on every module-running command, and can be
|
|
||||||
repeated:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup
|
|
||||||
```
|
|
||||||
|
|
||||||
For a folder, MVT loads its non-hidden top-level `*.py` files in sorted order
|
|
||||||
and skips `__init__.py`. `MVT_CUSTOM_MODULES` names a folder to load on every
|
|
||||||
module-running command, before any `--load-module` path:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip
|
|
||||||
```
|
|
||||||
|
|
||||||
Files loaded this way follow the same rules as packaged modules.
|
|
||||||
`--list-modules` reports them with the SHA-256 hash of the file in place of a
|
|
||||||
version. An editable install of the package (`pip install -e .`) also works:
|
|
||||||
the modules load through the entry point, and `mvt plugins list` shows the
|
|
||||||
package with the `local` origin.
|
|
||||||
|
|
||||||
Loading by path is for development. Move a module into a package once it
|
|
||||||
works.
|
|
||||||
|
|
||||||
## Auditing loaded modules
|
## Auditing loaded modules
|
||||||
|
|
||||||
Because installed module packages load automatically, MVT records where every
|
Because installed module packages load automatically, MVT records where every
|
||||||
@@ -373,9 +214,6 @@ module came from:
|
|||||||
- When a command runs with an `--output` folder, the `command.log` file
|
- When a command runs with an `--output` folder, the `command.log` file
|
||||||
records one line per module source with the source's version or hash and
|
records one line per module source with the source's version or hash and
|
||||||
the list of modules loaded from it.
|
the list of modules loaded from it.
|
||||||
- `mvt plugins list` lists the installed packages, where each of them
|
|
||||||
was installed from and how many modules it contributes, see
|
|
||||||
[Managing Plugins](plugins.md).
|
|
||||||
|
|
||||||
## Profiling
|
## Profiling
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ configuration:
|
|||||||
|
|
||||||
The exact parent folder follows the platform convention used for MVT's
|
The exact parent folder follows the platform convention used for MVT's
|
||||||
`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use
|
`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use
|
||||||
`plugin_config_path()` from `mvt.plugin` instead of building the path by hand.
|
`mvt.common.plugin_config.plugin_config_path()` instead of building the path by
|
||||||
|
hand.
|
||||||
|
|
||||||
Plugin names must be lowercase and may only contain letters, digits and dashes,
|
Plugin names must be lowercase and may only contain letters, digits and dashes,
|
||||||
matching the `mvt-plugin-<name>` package naming convention. MVT creates the
|
matching the `mvt-plugin-<name>` package naming convention. MVT creates the
|
||||||
@@ -37,8 +38,9 @@ leaves a partially written settings file behind.
|
|||||||
|
|
||||||
Everything else a plugin keeps on disk, such as a cache, a downloaded artifact
|
Everything else a plugin keeps on disk, such as a cache, a downloaded artifact
|
||||||
or synchronization state, belongs in the folder returned by the `data_folder()`
|
or synchronization state, belongs in the folder returned by the `data_folder()`
|
||||||
class method of the plugin's settings class, or by `plugin_data_folder()` from
|
class method of the plugin's settings class, or by
|
||||||
`mvt.plugin`, called with the plugin name if the plugin has no settings class:
|
`mvt.common.plugin_config.plugin_data_folder()` called with the plugin name if
|
||||||
|
the plugin has no settings class:
|
||||||
|
|
||||||
```
|
```
|
||||||
~/.local/share/mvt/plugin-data/<plugin name>/ # Linux
|
~/.local/share/mvt/plugin-data/<plugin name>/ # Linux
|
||||||
@@ -74,7 +76,7 @@ defaults:
|
|||||||
```python
|
```python
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from mvt.plugin import MVTPluginSettings
|
from mvt.common.plugin_config import MVTPluginSettings
|
||||||
|
|
||||||
|
|
||||||
class ExamplePluginSettings(MVTPluginSettings):
|
class ExamplePluginSettings(MVTPluginSettings):
|
||||||
@@ -92,15 +94,12 @@ from datetime import datetime, timezone
|
|||||||
|
|
||||||
import click
|
import click
|
||||||
|
|
||||||
from mvt.plugin import plugin_env_prefix
|
|
||||||
|
|
||||||
|
|
||||||
def sync():
|
def sync():
|
||||||
settings = ExamplePluginSettings.load()
|
settings = ExamplePluginSettings.load()
|
||||||
if not settings.API_KEY:
|
if not settings.API_KEY:
|
||||||
prefix = plugin_env_prefix(settings.plugin_name)
|
|
||||||
raise click.ClickException(
|
raise click.ClickException(
|
||||||
f"No API key configured. Set {prefix}API_KEY or "
|
"No API key configured. Set MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY or "
|
||||||
"run 'example-plugin configure'."
|
"run 'example-plugin configure'."
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -108,9 +107,6 @@ def sync():
|
|||||||
settings.save()
|
settings.save()
|
||||||
```
|
```
|
||||||
|
|
||||||
The message builds the variable name with `plugin_env_prefix()`, see
|
|
||||||
[Environment Variables](#environment-variables).
|
|
||||||
|
|
||||||
A missing settings file is not an error: the plugin then runs on the field
|
A missing settings file is not an error: the plugin then runs on the field
|
||||||
defaults and on whatever the environment provides. `save()` only persists the
|
defaults and on whatever the environment provides. `save()` only persists the
|
||||||
values that differ from the defaults, and it never touches MVT's `config.yaml`.
|
values that differ from the defaults, and it never touches MVT's `config.yaml`.
|
||||||
@@ -133,10 +129,6 @@ export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=...
|
|||||||
export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50
|
export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50
|
||||||
```
|
```
|
||||||
|
|
||||||
Do not repeat the plugin name in a field name: a plugin named `example-scanner`
|
|
||||||
with a `SCANNER_API_KEY` field asks the user for
|
|
||||||
`MVT_PLUGIN_EXAMPLE_SCANNER_SCANNER_API_KEY`. Name the field `API_KEY`.
|
|
||||||
|
|
||||||
Settings resolve in this order, from highest to lowest priority:
|
Settings resolve in this order, from highest to lowest priority:
|
||||||
|
|
||||||
1. Arguments passed to the settings class directly, such as
|
1. Arguments passed to the settings class directly, such as
|
||||||
|
|||||||
@@ -1,116 +0,0 @@
|
|||||||
# Managing Plugins
|
|
||||||
|
|
||||||
Plugin packages extend MVT with additional
|
|
||||||
[forensic modules](index.md#custom-modules) and
|
|
||||||
[CLI commands](custom_commands.md). Because installed packages load
|
|
||||||
automatically, `mvt plugins` audits what is installed and checks whether
|
|
||||||
updates are available. The command lives on `mvt` only, although the packages
|
|
||||||
it lists extend `mvt-ios` and `mvt-android` too.
|
|
||||||
|
|
||||||
## List Installed Plugins
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mvt plugins list
|
|
||||||
```
|
|
||||||
|
|
||||||
```
|
|
||||||
Installed MVT plugins
|
|
||||||
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓
|
|
||||||
┃ Name ┃ Version ┃ Origin ┃ Modules ┃ Commands ┃
|
|
||||||
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩
|
|
||||||
│ mvt-plugin-example │ 1.2.0 │ pypi │ 4 │ summarize │
|
|
||||||
│ mvt-plugin-research │ 0.1.0 │ git+3f9a1c7d │ 2 │ - │
|
|
||||||
│ mvt-plugin-local │ 0.0.1 │ local │ 1 │ triage │
|
|
||||||
└─────────────────────┴─────────┴──────────────┴─────────┴───────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
The origin records where each package was installed from: `pypi` for a package
|
|
||||||
installed from a package index, `git+<commit>` for a package installed directly
|
|
||||||
from a repository, and `local` for a package installed from a local folder or
|
|
||||||
archive rather than from an index, including an editable development install.
|
|
||||||
The last two columns show how many forensic modules the package contributes and
|
|
||||||
which CLI commands it adds.
|
|
||||||
|
|
||||||
A plugin whose modules cannot be imported is listed with `error` in the
|
|
||||||
`Modules` column rather than breaking the listing.
|
|
||||||
|
|
||||||
## Check for Updates
|
|
||||||
|
|
||||||
```bash
|
|
||||||
mvt plugins check-updates
|
|
||||||
```
|
|
||||||
|
|
||||||
```
|
|
||||||
Plugin updates available:
|
|
||||||
mvt-plugin-example 1.2.0 → 1.3.0
|
|
||||||
Upgrade with: pip install -U mvt-plugin-example
|
|
||||||
|
|
||||||
MVT does not install plugin updates. Run the command above when you decide to
|
|
||||||
upgrade.
|
|
||||||
```
|
|
||||||
|
|
||||||
Packages installed from a package index are compared against the latest release
|
|
||||||
published for them. A package which was never published, for example a plugin
|
|
||||||
distributed only within an organization, is skipped silently.
|
|
||||||
|
|
||||||
!!! note
|
|
||||||
|
|
||||||
Packages shown with the `pypi` origin are compared against
|
|
||||||
[PyPI](https://pypi.org), whichever index they were installed from. A
|
|
||||||
plugin installed from a private index under a name which also exists on
|
|
||||||
PyPI is therefore compared against the unrelated public package of that
|
|
||||||
name. Give plugins published to a private index a name which is not taken
|
|
||||||
on PyPI, and treat an unexpected update suggestion as a reason to check
|
|
||||||
where the package would come from.
|
|
||||||
|
|
||||||
!!! warning
|
|
||||||
|
|
||||||
MVT never installs or upgrades a plugin itself, it only prints the command
|
|
||||||
which does. Upgrading a plugin in the middle of an investigation changes
|
|
||||||
the modules producing the results, and a plugin runs as trusted code inside
|
|
||||||
the MVT process, so pulling in a new version is a decision for the analyst
|
|
||||||
to make deliberately and not a side effect of running a check.
|
|
||||||
|
|
||||||
## Automatic Update Checks
|
|
||||||
|
|
||||||
MVT also reports available plugin updates in the banner printed when a command
|
|
||||||
starts:
|
|
||||||
|
|
||||||
```
|
|
||||||
MVT - Mobile Verification Toolkit
|
|
||||||
|
|
||||||
https://mvt.re
|
|
||||||
Version: 2026.7.29
|
|
||||||
|
|
||||||
Plugin updates available:
|
|
||||||
mvt-plugin-example 1.2.0 → 1.3.0 (pip install -U mvt-plugin-example)
|
|
||||||
```
|
|
||||||
|
|
||||||
This check runs at most once every 12 hours. In between checks MVT prints the
|
|
||||||
findings of the latest check without contacting anything, so a plugin update
|
|
||||||
stays visible without a lookup on every command. The
|
|
||||||
`mvt plugins check-updates` command checks immediately, regardless of when the
|
|
||||||
last check happened.
|
|
||||||
|
|
||||||
The automatic check is skipped when the `--disable-update-check` option is
|
|
||||||
used, when `NETWORK_ACCESS_ALLOWED` is disabled in the MVT configuration, and
|
|
||||||
when no plugins are installed.
|
|
||||||
|
|
||||||
## Plugins Installed From a Repository
|
|
||||||
|
|
||||||
A plugin installed with `pip install "mvt-plugin-example @ git+<url>"` is
|
|
||||||
checked by asking the remote repository which commit the installed revision
|
|
||||||
points at now. MVT runs git and ssh in batch mode, so a repository which needs
|
|
||||||
credentials MVT does not already have fails the check instead of prompting for
|
|
||||||
them. The check is skipped silently when git is not available, when the
|
|
||||||
repository cannot be reached, and when access to it is denied.
|
|
||||||
|
|
||||||
How the plugin was installed decides what an update means:
|
|
||||||
|
|
||||||
- A plugin installed from a branch is reported as outdated when the branch has
|
|
||||||
moved past the installed commit.
|
|
||||||
- A plugin installed from a specific commit or a tag is pinned. It is never
|
|
||||||
reported as outdated, however far the branch it came from moves on.
|
|
||||||
|
|
||||||
Pinning a plugin to a commit or a tag is therefore the way to keep the modules
|
|
||||||
used across an investigation stable.
|
|
||||||
+1
-1
@@ -9,7 +9,7 @@ Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forens
|
|||||||
It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors.
|
It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors.
|
||||||
|
|
||||||
|
|
||||||
In this documentation you will find instructions on how to install and run the `mvt-ios`, `mvt-android` and `mvt` commands, and guidance on how to interpret the extracted results.
|
In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results.
|
||||||
|
|
||||||
## Resources
|
## Resources
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -64,7 +64,7 @@ It is recommended to try installing and running MVT from [Windows Subsystem Linu
|
|||||||
pipx install mvt
|
pipx install mvt
|
||||||
```
|
```
|
||||||
|
|
||||||
You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window.
|
You now should have the `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window.
|
||||||
|
|
||||||
### Installing from PyPI directly into a virtual environment
|
### Installing from PyPI directly into a virtual environment
|
||||||
You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool:
|
You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool:
|
||||||
@@ -84,7 +84,7 @@ source env/bin/activate
|
|||||||
pip install mvt
|
pip install mvt
|
||||||
```
|
```
|
||||||
|
|
||||||
The `mvt`, `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active.
|
The `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active.
|
||||||
|
|
||||||
### Installing from git source with pipx
|
### Installing from git source with pipx
|
||||||
If you want to have the latest features in development, you can install MVT directly from the source code in git.
|
If you want to have the latest features in development, you can install MVT directly from the source code in git.
|
||||||
@@ -93,7 +93,7 @@ If you want to have the latest features in development, you can install MVT dire
|
|||||||
pipx install --force git+https://github.com/mvt-project/mvt.git
|
pipx install --force git+https://github.com/mvt-project/mvt.git
|
||||||
```
|
```
|
||||||
|
|
||||||
You now should have the `mvt`, `mvt-ios` and `mvt-android` utilities installed.
|
You now should have the `mvt-ios` and `mvt-android` utilities installed.
|
||||||
|
|
||||||
**Notes:**
|
**Notes:**
|
||||||
1. The `--force` flag is necessary to force the reinstallation of the package.
|
1. The `--force` flag is necessary to force the reinstallation of the package.
|
||||||
|
|||||||
+1
-1
@@ -71,7 +71,7 @@ So far MVT implements only a subset of [STIX2 specifications](https://docs.oasis
|
|||||||
- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2).
|
- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2).
|
||||||
- We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns.
|
- We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns.
|
||||||
|
|
||||||
You can automatically download the latest public indicator files with the command `mvt download-iocs`. The per-platform forms `mvt-ios download-iocs` and `mvt-android download-iocs` do the same thing. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT.
|
You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT.
|
||||||
|
|
||||||
Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.
|
Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.
|
||||||
|
|
||||||
+14
-18
@@ -1,32 +1,30 @@
|
|||||||
# Check an iOS Sysdiagnose
|
# Check an iOS Sysdiagnose
|
||||||
|
|
||||||
`mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by
|
`mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by
|
||||||
custom MVT modules. MVT does not include built-in sysdiagnose modules. The
|
custom MVT modules. MVT does not include built-in sysdiagnose modules. You must
|
||||||
command runs the modules of the installed
|
load at least one custom module that explicitly supports this command.
|
||||||
[plugin packages](../development/index.md#installed-module-packages) which
|
|
||||||
declare support for it. Install at least one such package first.
|
|
||||||
|
|
||||||
The command accepts either an extracted sysdiagnose directory or the original
|
The command accepts either an extracted sysdiagnose directory or the original
|
||||||
gzip-compressed tar archive.
|
gzip-compressed tar archive.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mvt-ios check-sysdiagnose --output ./results \
|
mvt-ios check-sysdiagnose \
|
||||||
|
--load-module ./sysdiagnose_modules.py \
|
||||||
|
--output ./results \
|
||||||
./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz
|
./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `--hashes` to include hashes for analyzed files in `info.json`, and
|
Use `--hashes` to include hashes for analyzed files in `info.json`, and
|
||||||
`--list-modules` to display the eligible modules without running them.
|
`--list-modules` to display the eligible custom modules without running them.
|
||||||
|
|
||||||
## Writing a custom module
|
## Writing a custom module
|
||||||
|
|
||||||
Extend `SysdiagnoseExtraction` from `mvt.plugin`, see
|
Extend `SysdiagnoseExtraction` to access the archive contents consistently for
|
||||||
[Writing a module](../development/index.md#writing-a-module). The module reads
|
both directory and tar inputs. Each module must declare the command explicitly
|
||||||
the archive the same way whether MVT was given a folder or a tar archive. It
|
in `supported_commands`.
|
||||||
declares the command in `supported_commands`. While writing one,
|
|
||||||
[load it from its file](../development/index.md#developing-modules-locally).
|
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from mvt.plugin import SysdiagnoseExtraction
|
from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
|
||||||
|
|
||||||
|
|
||||||
class ExampleSysdiagnoseModule(SysdiagnoseExtraction):
|
class ExampleSysdiagnoseModule(SysdiagnoseExtraction):
|
||||||
@@ -46,9 +44,7 @@ class ExampleSysdiagnoseModule(SysdiagnoseExtraction):
|
|||||||
return None
|
return None
|
||||||
```
|
```
|
||||||
|
|
||||||
MVT extracts a tar archive first. It calls `from_sysdiagnose_folder()` on each
|
The base class provides `from_sysdiagnose_folder()` and
|
||||||
module before `run()`. `ips_files` lists the IPS crash reports.
|
`from_sysdiagnose_tar()` setup hooks, as well as protected file lookup, file
|
||||||
|
reading, and timezone extraction helpers. IPS crash-report metadata is exposed
|
||||||
`_get_files_by_pattern()` and `_get_file_content()` are internal helpers of the
|
on `ips_files`.
|
||||||
base class. Use them to read the archive. Their names and signatures can change
|
|
||||||
between releases. See `src/mvt/ios/modules/sysdiagnose/base.py`.
|
|
||||||
@@ -53,5 +53,4 @@ nav:
|
|||||||
- Development Instructions: "development/index.md"
|
- Development Instructions: "development/index.md"
|
||||||
- Custom CLI Commands: "development/custom_commands.md"
|
- Custom CLI Commands: "development/custom_commands.md"
|
||||||
- Plugin Configuration: "development/plugin_configuration.md"
|
- Plugin Configuration: "development/plugin_configuration.md"
|
||||||
- Managing Plugins: "development/plugins.md"
|
|
||||||
- License: "license.md"
|
- License: "license.md"
|
||||||
+4
-7
@@ -24,8 +24,7 @@ dependencies = [
|
|||||||
"simplejson==4.1.1",
|
"simplejson==4.1.1",
|
||||||
"packaging==26.3",
|
"packaging==26.3",
|
||||||
"appdirs==1.4.4",
|
"appdirs==1.4.4",
|
||||||
"iphone_backup_decrypt==0.9.0",
|
"iOSbackup==0.9.925",
|
||||||
"pycryptodome>=3.20.0",
|
|
||||||
"adb-shell[usb]==0.4.4",
|
"adb-shell[usb]==0.4.4",
|
||||||
"libusb1==3.4.0",
|
"libusb1==3.4.0",
|
||||||
"cryptography==50.0.0",
|
"cryptography==50.0.0",
|
||||||
@@ -45,7 +44,6 @@ homepage = "https://docs.mvt.re/en/latest/"
|
|||||||
repository = "https://github.com/mvt-project/mvt"
|
repository = "https://github.com/mvt-project/mvt"
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
mvt = "mvt.cli:main"
|
|
||||||
mvt-ios = "mvt.ios:main"
|
mvt-ios = "mvt.ios:main"
|
||||||
mvt-android = "mvt.android:main"
|
mvt-android = "mvt.android:main"
|
||||||
|
|
||||||
@@ -73,7 +71,7 @@ docs = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["setuptools>=61.0", "setuptools-scm>=8"]
|
requires = ["setuptools>=61.0"]
|
||||||
build-backend = "setuptools.build_meta"
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
[tool.coverage.run]
|
[tool.coverage.run]
|
||||||
@@ -122,6 +120,5 @@ where = ["src"]
|
|||||||
[tool.setuptools.package-data]
|
[tool.setuptools.package-data]
|
||||||
mvt = ["ios/data/*.json"]
|
mvt = ["ios/data/*.json"]
|
||||||
|
|
||||||
[tool.setuptools_scm]
|
[tool.setuptools.dynamic]
|
||||||
# The version is the latest v* tag; ignore the archive/* tags.
|
version = { attr = "mvt.common.version.MVT_VERSION" }
|
||||||
git_describe_command = "git describe --dirty --tags --long --match 'v*'"
|
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
@@ -21,10 +20,10 @@ class DumpsysAccessibilityArtifact(AndroidArtifact):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(
|
||||||
f'Found accessibility service: "{result["component"]}"',
|
f'Found accessibility service: "{result["service"]}"',
|
||||||
"",
|
"",
|
||||||
result,
|
result,
|
||||||
)
|
)
|
||||||
|
|
||||||
def parse(self, content: str) -> None:
|
def parse(self, content: str) -> None:
|
||||||
"""
|
"""
|
||||||
@@ -34,69 +33,41 @@ class DumpsysAccessibilityArtifact(AndroidArtifact):
|
|||||||
:param content: content of the accessibility section (string)
|
:param content: content of the accessibility section (string)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self.results: list[dict[str, Any]] = []
|
# "Old" syntax
|
||||||
services: dict[tuple[int | None, str], dict] = {}
|
in_services = False
|
||||||
user_id: int | None = None
|
for line in content.splitlines():
|
||||||
state: str | None = None
|
if line.strip().startswith("installed services:"):
|
||||||
|
in_services = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not in_services:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if line.strip() == "}":
|
||||||
|
# At end of installed services
|
||||||
|
break
|
||||||
|
|
||||||
|
service = line.split(":")[1].strip()
|
||||||
|
|
||||||
|
self.results.append(
|
||||||
|
{
|
||||||
|
"package_name": service.split("/")[0],
|
||||||
|
"service": service,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# "New" syntax - AOSP >= 14 (?)
|
||||||
|
# Looks like:
|
||||||
|
# Enabled services:{{com.azure.authenticator/com.microsoft.brooklyn.module.accessibility.BrooklynAccessibilityService}, {com.agilebits.onepassword/com.agilebits.onepassword.filling.accessibility.FillingAccessibilityService}}
|
||||||
|
|
||||||
for line in content.splitlines():
|
for line in content.splitlines():
|
||||||
user_match = re.search(r"attributes:\{id=(\d+)", line)
|
if line.strip().startswith("Enabled services:"):
|
||||||
if user_match:
|
matches = re.finditer(r"{([^{]+?)}", line)
|
||||||
user_id = int(user_match.group(1))
|
|
||||||
|
|
||||||
stripped = line.strip()
|
for match in matches:
|
||||||
state_match = re.match(
|
# Each match is in format: <package_name>/<service>
|
||||||
r"(?i)(installed|enabled|binding|bound|crashed) services\s*:\s*\{(.*)",
|
package_name, _, service = match.group(1).partition("/")
|
||||||
stripped,
|
|
||||||
)
|
self.results.append(
|
||||||
if state_match:
|
{"package_name": package_name, "service": service}
|
||||||
state = state_match.group(1).lower()
|
|
||||||
inline = state_match.group(2)
|
|
||||||
for component in re.findall(
|
|
||||||
r"\{?([\w.$-]+/[\w.$-]+)(?:\s+\(A11yTool\))?\}?", inline
|
|
||||||
):
|
|
||||||
service = services.setdefault(
|
|
||||||
(user_id, component), self._new_service(component, user_id)
|
|
||||||
)
|
)
|
||||||
service[self._state_field(state)] = True
|
|
||||||
service["accessibility_tool"] = "(A11yTool)" in inline
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not state:
|
|
||||||
continue
|
|
||||||
if stripped == "}" or stripped.startswith("AccessibilityInputFilter"):
|
|
||||||
state = None
|
|
||||||
continue
|
|
||||||
component_match = re.search(
|
|
||||||
r"(?:\d+\s*:\s*)?([\w.$-]+/[\w.$-]+)(?:\s+\(A11yTool\))?",
|
|
||||||
stripped,
|
|
||||||
)
|
|
||||||
if component_match:
|
|
||||||
component = component_match.group(1)
|
|
||||||
service = services.setdefault(
|
|
||||||
(user_id, component), self._new_service(component, user_id)
|
|
||||||
)
|
|
||||||
service[self._state_field(state)] = True
|
|
||||||
service["accessibility_tool"] = "(A11yTool)" in stripped
|
|
||||||
|
|
||||||
self.results.extend(services.values())
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _state_field(state: str) -> str:
|
|
||||||
return {"binding": "binding", "bound": "bound"}.get(state, state)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _new_service(component: str, user_id: int | None) -> dict:
|
|
||||||
package_name, service_name = component.split("/", 1)
|
|
||||||
return {
|
|
||||||
"user_id": user_id,
|
|
||||||
"component": component,
|
|
||||||
"package_name": package_name,
|
|
||||||
"service_name": service_name,
|
|
||||||
"installed": False,
|
|
||||||
"enabled": False,
|
|
||||||
"binding": False,
|
|
||||||
"bound": False,
|
|
||||||
"crashed": False,
|
|
||||||
"accessibility_tool": False,
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,6 @@
|
|||||||
import base64
|
import base64
|
||||||
import binascii
|
import binascii
|
||||||
import hashlib
|
import hashlib
|
||||||
import re
|
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
@@ -99,34 +98,6 @@ class DumpsysADBArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
return keystore
|
return keystore
|
||||||
|
|
||||||
def parse_binary_xml(self, data: bytes) -> list[dict]:
|
|
||||||
"""Recover ADB key records from Android binary XML (ABX).
|
|
||||||
|
|
||||||
Some dumpstate implementations embed ABX in a text stream and replace
|
|
||||||
binary token bytes. The public key remains intact, while unavailable
|
|
||||||
numeric metadata is represented as ``None`` rather than corrupt text.
|
|
||||||
"""
|
|
||||||
keystore = []
|
|
||||||
seen = set()
|
|
||||||
for match in re.finditer(
|
|
||||||
rb"(?<![A-Za-z0-9+/])([A-Za-z0-9+/]{300,}={0,2})"
|
|
||||||
rb"(?: ([A-Za-z0-9_.@-]+))?",
|
|
||||||
data,
|
|
||||||
):
|
|
||||||
key = match.group(1)
|
|
||||||
try:
|
|
||||||
base64.b64decode(key, validate=True)
|
|
||||||
except (binascii.Error, ValueError):
|
|
||||||
continue
|
|
||||||
if key in seen:
|
|
||||||
continue
|
|
||||||
seen.add(key)
|
|
||||||
full_key = key + (b" " + match.group(2) if match.group(2) else b"")
|
|
||||||
key_info = self.calculate_key_info(full_key)
|
|
||||||
key_info["last_connected"] = None
|
|
||||||
keystore.append(key_info)
|
|
||||||
return keystore
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def calculate_key_info(user_key: bytes) -> dict:
|
def calculate_key_info(user_key: bytes) -> dict:
|
||||||
if b" " in user_key:
|
if b" " in user_key:
|
||||||
@@ -147,7 +118,7 @@ class DumpsysADBArtifact(AndroidArtifact):
|
|||||||
return {
|
return {
|
||||||
"user": user.decode("utf-8"),
|
"user": user.decode("utf-8"),
|
||||||
"fingerprint": key_fingerprint_colon,
|
"fingerprint": key_fingerprint_colon,
|
||||||
"key": key_base64.decode("ascii", errors="replace"),
|
"key": key_base64,
|
||||||
}
|
}
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
@@ -204,23 +175,12 @@ class DumpsysADBArtifact(AndroidArtifact):
|
|||||||
# Keystore is in XML format on some devices and we need to parse it
|
# Keystore is in XML format on some devices and we need to parse it
|
||||||
if keystore_data and keystore_data.startswith(b"<?xml"):
|
if keystore_data and keystore_data.startswith(b"<?xml"):
|
||||||
parsed["debugging_manager"]["keystore"] = self.parse_xml(keystore_data)
|
parsed["debugging_manager"]["keystore"] = self.parse_xml(keystore_data)
|
||||||
elif keystore_data and keystore_data.startswith(b"ABX\x00"):
|
|
||||||
parsed["debugging_manager"]["keystore"] = self.parse_binary_xml(
|
|
||||||
keystore_data
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
# Keystore is not XML format
|
# Keystore is not XML format
|
||||||
parsed["debugging_manager"]["keystore"] = keystore_data
|
parsed["debugging_manager"]["keystore"] = keystore_data
|
||||||
|
|
||||||
parsed = parsed["debugging_manager"]
|
parsed = parsed["debugging_manager"]
|
||||||
|
|
||||||
for key, value in list(parsed.items()):
|
|
||||||
if isinstance(value, bytes):
|
|
||||||
decoded = value.decode("utf-8", errors="replace")
|
|
||||||
parsed[key] = (
|
|
||||||
decoded == "true" if decoded in ("true", "false") else decoded
|
|
||||||
)
|
|
||||||
|
|
||||||
# Calculate key fingerprints for better readability
|
# Calculate key fingerprints for better readability
|
||||||
key_info = []
|
key_info = []
|
||||||
for user_key in parsed.get("user_keys", []):
|
for user_key in parsed.get("user_keys", []):
|
||||||
|
|||||||
@@ -4,7 +4,6 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import re
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
||||||
@@ -28,14 +27,14 @@ class DumpsysAppopsArtifact(AndroidArtifact):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
for entry in perm["entries"]:
|
for entry in perm["entries"]:
|
||||||
if entry.get("timestamp"):
|
if "timestamp" in entry:
|
||||||
records.append(
|
records.append(
|
||||||
{
|
{
|
||||||
"timestamp": entry["timestamp"],
|
"timestamp": entry["timestamp"],
|
||||||
"module": self.__class__.__name__,
|
"module": self.__class__.__name__,
|
||||||
"event": entry["event"],
|
"event": entry["access"],
|
||||||
"data": f"{result['package_name']} access to "
|
"data": f"{result['package_name']} access to "
|
||||||
f"{perm['name']}: {entry['event']}",
|
f"{perm['name']}: {entry['access']}",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -52,7 +51,7 @@ class DumpsysAppopsArtifact(AndroidArtifact):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
# We use a placeholder entry to create a basic alert even without permission entries.
|
# We use a placeholder entry to create a basic alert even without permission entries.
|
||||||
placeholder_entry = {"event": "unknown", "timestamp": ""}
|
placeholder_entry = {"access": "Unknown", "timestamp": ""}
|
||||||
|
|
||||||
for perm in result["permissions"]:
|
for perm in result["permissions"]:
|
||||||
if (
|
if (
|
||||||
@@ -61,12 +60,12 @@ class DumpsysAppopsArtifact(AndroidArtifact):
|
|||||||
):
|
):
|
||||||
for entry in sorted(
|
for entry in sorted(
|
||||||
perm["entries"] or [placeholder_entry],
|
perm["entries"] or [placeholder_entry],
|
||||||
key=lambda x: x.get("timestamp") or "",
|
key=lambda x: x["timestamp"],
|
||||||
):
|
):
|
||||||
cleaned_result = result.copy()
|
cleaned_result = result.copy()
|
||||||
cleaned_result["permissions"] = [perm]
|
cleaned_result["permissions"] = [perm]
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(
|
||||||
f"Package '{result['package_name']}' had risky permission '{perm['name']}' set to '{entry['event']}' at {entry['timestamp']}",
|
f"Package '{result['package_name']}' had risky permission '{perm['name']}' set to '{entry['access']}' at {entry['timestamp']}",
|
||||||
entry["timestamp"],
|
entry["timestamp"],
|
||||||
cleaned_result,
|
cleaned_result,
|
||||||
)
|
)
|
||||||
@@ -74,151 +73,111 @@ class DumpsysAppopsArtifact(AndroidArtifact):
|
|||||||
elif result["package_name"] in RISKY_PACKAGES:
|
elif result["package_name"] in RISKY_PACKAGES:
|
||||||
for entry in sorted(
|
for entry in sorted(
|
||||||
perm["entries"] or [placeholder_entry],
|
perm["entries"] or [placeholder_entry],
|
||||||
key=lambda x: x.get("timestamp") or "",
|
key=lambda x: x["timestamp"],
|
||||||
):
|
):
|
||||||
cleaned_result = result.copy()
|
cleaned_result = result.copy()
|
||||||
cleaned_result["permissions"] = [perm]
|
cleaned_result["permissions"] = [perm]
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(
|
||||||
f"Risky package '{result['package_name']}' had '{perm['name']}' permission set to '{entry['event']}' at {entry['timestamp']}",
|
f"Risky package '{result['package_name']}' had '{perm['name']}' permission set to '{entry['access']}' at {entry['timestamp']}",
|
||||||
entry["timestamp"],
|
entry["timestamp"],
|
||||||
cleaned_result,
|
cleaned_result,
|
||||||
)
|
)
|
||||||
|
|
||||||
def parse(self, output: str) -> None:
|
def parse(self, output: str) -> None:
|
||||||
self.results: list[dict[str, Any]] = []
|
# self.results: List[Dict[str, Any]] = []
|
||||||
permission: dict[str, Any] | None = None
|
perm: dict[str, Any] = {}
|
||||||
package: dict[str, Any] | None = None
|
package: dict[str, Any] = {}
|
||||||
uid: str | None = None
|
entry: dict[str, Any] = {}
|
||||||
uid_details: dict[str, Any] = {}
|
uid = None
|
||||||
attribution: str | None = None
|
|
||||||
in_packages = False
|
in_packages = False
|
||||||
|
|
||||||
def finish_permission() -> None:
|
|
||||||
nonlocal permission
|
|
||||||
if package is not None and permission is not None:
|
|
||||||
package["permissions"].append(permission)
|
|
||||||
permission = None
|
|
||||||
|
|
||||||
def finish_package() -> None:
|
|
||||||
nonlocal package
|
|
||||||
finish_permission()
|
|
||||||
if package is not None:
|
|
||||||
self.results.append(package)
|
|
||||||
package = None
|
|
||||||
|
|
||||||
for line in output.splitlines():
|
for line in output.splitlines():
|
||||||
uid_match = re.match(r"^ Uid ([^:]+):$", line)
|
if line.startswith(" Uid 0:"):
|
||||||
if uid_match:
|
|
||||||
in_packages = True
|
in_packages = True
|
||||||
finish_package()
|
|
||||||
uid = uid_match.group(1)
|
|
||||||
uid_details = {
|
|
||||||
"uid_state": None,
|
|
||||||
"capability": None,
|
|
||||||
"app_widget_visible": None,
|
|
||||||
"default_modes": {},
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not in_packages:
|
if not in_packages:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
uid_property = re.match(
|
if line.startswith(" Uid "):
|
||||||
r"^ (state|capability|appWidgetVisible)=(.*)$", line
|
uid = line[6:-1]
|
||||||
)
|
if entry:
|
||||||
if uid_property:
|
perm["entries"].append(entry)
|
||||||
key = {
|
entry = {}
|
||||||
"state": "uid_state",
|
if package:
|
||||||
"appWidgetVisible": "app_widget_visible",
|
if perm:
|
||||||
}.get(uid_property.group(1), uid_property.group(1))
|
package["permissions"].append(perm)
|
||||||
value: Any = uid_property.group(2)
|
|
||||||
if value in ("true", "false"):
|
|
||||||
value = value == "true"
|
|
||||||
uid_details[key] = value
|
|
||||||
continue
|
|
||||||
|
|
||||||
default_mode = re.match(r"^ ([A-Z0-9_]+): mode=([^\s]+)", line)
|
perm = {}
|
||||||
if default_mode and package is None:
|
self.results.append(package)
|
||||||
uid_details["default_modes"][default_mode.group(1)] = (
|
package = {}
|
||||||
default_mode.group(2)
|
|
||||||
)
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if line.startswith(" Package "):
|
if line.startswith(" Package "):
|
||||||
finish_package()
|
if entry:
|
||||||
|
perm["entries"].append(entry)
|
||||||
|
entry = {}
|
||||||
|
|
||||||
|
if package:
|
||||||
|
if perm:
|
||||||
|
package["permissions"].append(perm)
|
||||||
|
|
||||||
|
perm = {}
|
||||||
|
self.results.append(package)
|
||||||
|
|
||||||
package = {
|
package = {
|
||||||
"package_name": line[12:-1],
|
"package_name": line[12:-1],
|
||||||
"permissions": [],
|
"permissions": [],
|
||||||
"uid": uid,
|
"uid": uid,
|
||||||
**uid_details,
|
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
|
|
||||||
operation_match = re.match(
|
if package and line.startswith(" ") and line[6] != " ":
|
||||||
r"^ ([A-Z0-9_]+)(?: \(([^)]+)\))?:\s*$", line
|
if entry:
|
||||||
)
|
perm["entries"].append(entry)
|
||||||
if package is not None and operation_match:
|
entry = {}
|
||||||
finish_permission()
|
if perm:
|
||||||
permission = {
|
package["permissions"].append(perm)
|
||||||
"name": operation_match.group(1),
|
perm = {}
|
||||||
"mode": operation_match.group(2),
|
|
||||||
"entries": [],
|
perm["name"] = line.split()[0]
|
||||||
}
|
perm["entries"] = []
|
||||||
attribution = None
|
if len(line.split()) > 1:
|
||||||
|
perm["access"] = line.split()[1][1:-2]
|
||||||
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
attribution_match = re.match(r"^\s{8,}([^=]+)=\[$", line)
|
if line.startswith(" "):
|
||||||
if attribution_match:
|
# Permission entry like:
|
||||||
attribution = attribution_match.group(1).strip()
|
# Reject: [fg-s]2021-05-19 22:02:52.054 (-314d1h25m2s33ms)
|
||||||
continue
|
access_type = line.split(":")[0].strip()
|
||||||
if line.strip() == "]":
|
if access_type not in ["Access", "Reject"]:
|
||||||
attribution = None
|
# Skipping invalid access type. Some entries are not in the format we expect
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if permission is None:
|
if entry:
|
||||||
continue
|
perm["entries"].append(entry)
|
||||||
event_match = re.match(
|
entry = {}
|
||||||
r"^\s*(Access|Reject):\s*\[([^]]+)\]\s*"
|
|
||||||
r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)\s*"
|
entry["access"] = access_type
|
||||||
r"(\([^)]*\))?(?:\s+duration=([^\s]+))?",
|
entry["type"] = line[line.find("[") + 1 : line.find("]")]
|
||||||
line,
|
|
||||||
)
|
try:
|
||||||
running_match = re.match(
|
entry["timestamp"] = convert_datetime_to_iso(
|
||||||
r"^\s*Running start at:\s*(\S+(?: \S+)?)",
|
datetime.strptime(
|
||||||
line,
|
line[line.find("]") + 1 : line.find("(")].strip(),
|
||||||
)
|
"%Y-%m-%d %H:%M:%S.%f",
|
||||||
if event_match:
|
)
|
||||||
entry = {
|
|
||||||
"event": event_match.group(1).lower(),
|
|
||||||
"access": event_match.group(1),
|
|
||||||
"uid_state": event_match.group(2),
|
|
||||||
"timestamp": convert_datetime_to_iso(
|
|
||||||
datetime.strptime(event_match.group(3), "%Y-%m-%d %H:%M:%S.%f")
|
|
||||||
),
|
|
||||||
"relative_time": event_match.group(4),
|
|
||||||
"duration": event_match.group(5),
|
|
||||||
"attribution": attribution,
|
|
||||||
}
|
|
||||||
permission["entries"].append(entry)
|
|
||||||
elif running_match:
|
|
||||||
raw_start = running_match.group(1)
|
|
||||||
timestamp = None
|
|
||||||
if re.fullmatch(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+", raw_start):
|
|
||||||
timestamp = convert_datetime_to_iso(
|
|
||||||
datetime.strptime(raw_start, "%Y-%m-%d %H:%M:%S.%f")
|
|
||||||
)
|
)
|
||||||
permission["entries"].append(
|
except ValueError:
|
||||||
{
|
# Invalid date format
|
||||||
"event": "running",
|
pass
|
||||||
"access": "Running",
|
|
||||||
"uid_state": None,
|
|
||||||
"timestamp": timestamp,
|
|
||||||
"relative_time": raw_start
|
|
||||||
if raw_start.startswith("+")
|
|
||||||
else None,
|
|
||||||
"duration": None,
|
|
||||||
"attribution": attribution,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
finish_package()
|
if line.strip() == "":
|
||||||
|
break
|
||||||
|
|
||||||
|
if entry:
|
||||||
|
perm["entries"].append(entry)
|
||||||
|
if perm:
|
||||||
|
package["permissions"].append(perm)
|
||||||
|
if package:
|
||||||
|
self.results.append(package)
|
||||||
@@ -18,18 +18,18 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact):
|
|||||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||||
action = record.get("action", "update")
|
action = record.get("action", "update")
|
||||||
package_name = record["package_name"]
|
package_name = record["package_name"]
|
||||||
vers = record["version_code"]
|
vers = record["vers"]
|
||||||
|
|
||||||
if vers == 0:
|
if vers == "0":
|
||||||
data = f"Recorded uninstall of package {package_name} (vers 0)"
|
data = f"Recorded uninstall of package {package_name} (vers 0)"
|
||||||
elif action == "downgrade":
|
elif action == "downgrade":
|
||||||
prev_vers = record.get("previous_version_code", "unknown")
|
prev_vers = record.get("previous_vers", "unknown")
|
||||||
data = f"Recorded downgrade of package {package_name} from vers {prev_vers} to vers {vers}"
|
data = f"Recorded downgrade of package {package_name} from vers {prev_vers} to vers {vers}"
|
||||||
else:
|
else:
|
||||||
data = f"Recorded update of package {package_name} with vers {vers}"
|
data = f"Recorded update of package {package_name} with vers {vers}"
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"timestamp": record["period_start"],
|
"timestamp": record["from"],
|
||||||
"module": self.__class__.__name__,
|
"module": self.__class__.__name__,
|
||||||
"event": "battery_daily",
|
"event": "battery_daily",
|
||||||
"data": data,
|
"data": data,
|
||||||
@@ -48,7 +48,6 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
def parse(self, output: str) -> None:
|
def parse(self, output: str) -> None:
|
||||||
self.results = []
|
|
||||||
daily = None
|
daily = None
|
||||||
daily_updates: list[dict[str, Any]] = []
|
daily_updates: list[dict[str, Any]] = []
|
||||||
records: list[dict[str, Any]] = []
|
records: list[dict[str, Any]] = []
|
||||||
@@ -60,10 +59,7 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
timeframe = line[13:].strip()
|
timeframe = line[13:].strip()
|
||||||
date_from, date_to = timeframe.strip(":").split(" to ", 1)
|
date_from, date_to = timeframe.strip(":").split(" to ", 1)
|
||||||
daily = {
|
daily = {"from": date_from[0:10], "to": date_to[0:10]}
|
||||||
"period_start": self._format_daily_timestamp(date_from),
|
|
||||||
"period_end": self._format_daily_timestamp(date_to),
|
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not daily:
|
if not daily:
|
||||||
@@ -74,30 +70,21 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
line = line.strip().replace("Update ", "")
|
line = line.strip().replace("Update ", "")
|
||||||
package_name, vers = line.split(" ", 1)
|
package_name, vers = line.split(" ", 1)
|
||||||
vers_raw = vers.split("=", 1)[1]
|
vers_nr = vers.split("=", 1)[1]
|
||||||
try:
|
|
||||||
version_code: int | str = int(vers_raw)
|
|
||||||
except ValueError:
|
|
||||||
version_code = vers_raw
|
|
||||||
|
|
||||||
already_seen = False
|
already_seen = False
|
||||||
for update in daily_updates:
|
for update in daily_updates:
|
||||||
if (
|
if package_name == update["package_name"] and vers_nr == update["vers"]:
|
||||||
package_name == update["package_name"]
|
|
||||||
and version_code == update["version_code"]
|
|
||||||
):
|
|
||||||
update["occurrences"] += 1
|
|
||||||
already_seen = True
|
already_seen = True
|
||||||
break
|
break
|
||||||
|
|
||||||
if not already_seen:
|
if not already_seen:
|
||||||
update_record: dict[str, Any] = {
|
update_record: dict[str, Any] = {
|
||||||
"action": "update",
|
"action": "update",
|
||||||
"period_start": daily["period_start"],
|
"from": daily["from"],
|
||||||
"period_end": daily["period_end"],
|
"to": daily["to"],
|
||||||
"package_name": package_name,
|
"package_name": package_name,
|
||||||
"version_code": version_code,
|
"vers": vers_nr,
|
||||||
"occurrences": 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
daily_updates.append(update_record)
|
daily_updates.append(update_record)
|
||||||
@@ -108,31 +95,26 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact):
|
|||||||
self._detect_uninstalls_and_downgrades(records)
|
self._detect_uninstalls_and_downgrades(records)
|
||||||
self.results.extend(records)
|
self.results.extend(records)
|
||||||
|
|
||||||
@staticmethod
|
def _detect_uninstalls_and_downgrades(
|
||||||
def _format_daily_timestamp(value: str) -> str:
|
self, records: list[dict[str, Any]]
|
||||||
if len(value) >= 19 and value[10] == "-":
|
) -> None:
|
||||||
return f"{value[:10]} {value[11:].replace('-', ':')}"
|
|
||||||
return value
|
|
||||||
|
|
||||||
def _detect_uninstalls_and_downgrades(self, records: list[dict[str, Any]]) -> None:
|
|
||||||
package_versions: dict[str, int] = {}
|
package_versions: dict[str, int] = {}
|
||||||
|
|
||||||
for record in sorted(
|
for record in sorted(
|
||||||
records,
|
records,
|
||||||
key=lambda record: (
|
key=lambda record: (
|
||||||
record["period_start"],
|
record["from"],
|
||||||
record["period_end"],
|
record["to"],
|
||||||
record["package_name"],
|
record["package_name"],
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
package_name = record["package_name"]
|
package_name = record["package_name"]
|
||||||
vers_nr = record["version_code"]
|
vers_nr = record["vers"]
|
||||||
|
|
||||||
if vers_nr == 0:
|
if vers_nr == "0":
|
||||||
record["action"] = "uninstall"
|
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(
|
||||||
f"Detected uninstall of package {package_name} (vers 0)",
|
f"Detected uninstall of package {package_name} (vers 0)",
|
||||||
record["period_start"],
|
record["from"],
|
||||||
record,
|
record,
|
||||||
)
|
)
|
||||||
package_versions.pop(package_name, None)
|
package_versions.pop(package_name, None)
|
||||||
@@ -146,11 +128,11 @@ class DumpsysBatteryDailyArtifact(AndroidArtifact):
|
|||||||
previous_vers = package_versions.get(package_name)
|
previous_vers = package_versions.get(package_name)
|
||||||
if previous_vers is not None and current_vers < previous_vers:
|
if previous_vers is not None and current_vers < previous_vers:
|
||||||
record["action"] = "downgrade"
|
record["action"] = "downgrade"
|
||||||
record["previous_version_code"] = previous_vers
|
record["previous_vers"] = str(previous_vers)
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(
|
||||||
f"Detected downgrade of package {package_name} "
|
f"Detected downgrade of package {package_name} "
|
||||||
f"from vers {previous_vers} to vers {current_vers}",
|
f"from vers {previous_vers} to vers {current_vers}",
|
||||||
record["period_start"],
|
record["from"],
|
||||||
record,
|
record,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,177 +1,96 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
# Mobile Verification Toolkit (MVT)
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import datetime
|
|
||||||
import re
|
|
||||||
|
|
||||||
from mvt.common.utils import convert_datetime_to_iso
|
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
|
|
||||||
class DumpsysBatteryHistoryArtifact(AndroidArtifact):
|
class DumpsysBatteryHistoryArtifact(AndroidArtifact):
|
||||||
"""Parser for package-related events in dumpsys batterystats history."""
|
"""
|
||||||
|
Parser for dumpsys dattery history events.
|
||||||
|
"""
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
for result in self.results:
|
for result in self.results:
|
||||||
package_name = result.get("package_name")
|
ioc_match = self.indicators.check_app_id(result["package_name"])
|
||||||
if not package_name:
|
|
||||||
continue
|
|
||||||
ioc_match = self.indicators.check_app_id(package_name)
|
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
)
|
)
|
||||||
|
continue
|
||||||
@staticmethod
|
|
||||||
def _parse_wall_time(value: str) -> datetime.datetime | None:
|
|
||||||
if re.fullmatch(r"\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+", value):
|
|
||||||
value = f"1900-{value}"
|
|
||||||
for date_format in (
|
|
||||||
"%Y-%m-%d-%H-%M-%S-%f",
|
|
||||||
"%Y-%m-%d-%H-%M-%S",
|
|
||||||
"%Y-%m-%d %H:%M:%S.%f",
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
return datetime.datetime.strptime(value, date_format)
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
return None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _elapsed_seconds(value: str) -> float | None:
|
|
||||||
if not value.startswith("+"):
|
|
||||||
return None
|
|
||||||
units = {"d": 86400, "h": 3600, "m": 60, "s": 1, "ms": 0.001}
|
|
||||||
total = 0.0
|
|
||||||
for number, unit in re.findall(r"(\d+)(ms|d|h|m|s)", value):
|
|
||||||
total += int(number) * units[unit]
|
|
||||||
return total
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _package_from_name(name: str) -> str | None:
|
|
||||||
clean = name.removeprefix("*walarm*:").removeprefix("*alarm*:")
|
|
||||||
slash_parts = clean.split("/")
|
|
||||||
if len(slash_parts) > 1:
|
|
||||||
first = slash_parts[0].lstrip("@")
|
|
||||||
if first.startswith(("com.", "org.", "net.")):
|
|
||||||
return first
|
|
||||||
for part in reversed(slash_parts[1:]):
|
|
||||||
candidate = part.lstrip("@").split(":", 1)[0]
|
|
||||||
if candidate.startswith(("com.", "org.", "net.")):
|
|
||||||
return candidate
|
|
||||||
parts = clean.split(".")
|
|
||||||
package_parts = []
|
|
||||||
for part in parts:
|
|
||||||
if part and part[0].islower():
|
|
||||||
package_parts.append(part)
|
|
||||||
else:
|
|
||||||
break
|
|
||||||
return ".".join(package_parts) if len(package_parts) >= 2 else None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _normalize_service(service: str) -> str:
|
|
||||||
# WorkManager decorates jobs with one or more scheduler prefixes.
|
|
||||||
if "@" in service:
|
|
||||||
candidates = [part for part in service.split("@") if "/" in part]
|
|
||||||
if candidates:
|
|
||||||
return candidates[-1]
|
|
||||||
return service
|
|
||||||
|
|
||||||
def parse(self, data: str) -> None:
|
def parse(self, data: str) -> None:
|
||||||
self.results: list[dict[str, str | None]] = []
|
|
||||||
anchor_time: datetime.datetime | None = None
|
|
||||||
anchor_elapsed = 0.0
|
|
||||||
has_history_heading = any(
|
|
||||||
line.startswith("Battery History") for line in data.splitlines()
|
|
||||||
)
|
|
||||||
in_history = not has_history_heading
|
|
||||||
|
|
||||||
for line in data.splitlines():
|
for line in data.splitlines():
|
||||||
stripped = line.strip()
|
if line.startswith("Battery History "):
|
||||||
if line.startswith("Battery History"):
|
|
||||||
if in_history:
|
|
||||||
break
|
|
||||||
in_history = True
|
|
||||||
continue
|
continue
|
||||||
if not in_history:
|
|
||||||
continue
|
if line.strip() == "":
|
||||||
if has_history_heading and not stripped:
|
|
||||||
break
|
break
|
||||||
reset_match = re.search(r"(?:RESET:)?TIME:\s*(\S+)", stripped)
|
|
||||||
if reset_match:
|
|
||||||
parsed_time = self._parse_wall_time(reset_match.group(1))
|
|
||||||
if parsed_time is not None:
|
|
||||||
elapsed_token = stripped.split()[0]
|
|
||||||
anchor_elapsed = self._elapsed_seconds(elapsed_token) or 0.0
|
|
||||||
anchor_time = parsed_time
|
|
||||||
continue
|
|
||||||
|
|
||||||
fields = stripped.split()
|
time_parts = line.strip().split()
|
||||||
if not fields:
|
time_elapsed = time_parts[0]
|
||||||
continue
|
if (
|
||||||
if len(fields) > 1 and re.fullmatch(r"\d{2}-\d{2}", fields[0]):
|
len(time_parts) > 1
|
||||||
time_elapsed = " ".join(fields[:2])
|
and len(time_parts[0]) == 5
|
||||||
line_time = self._parse_wall_time(time_elapsed)
|
and time_parts[0][2] == "-"
|
||||||
elapsed = None
|
and ":" in time_parts[1]
|
||||||
|
):
|
||||||
|
time_elapsed = " ".join(time_parts[:2])
|
||||||
|
|
||||||
|
event = ""
|
||||||
|
if line.find("+job") > 0:
|
||||||
|
event = "start_job"
|
||||||
|
payload = line.split("+job=", 1)[1]
|
||||||
|
uid, separator, service = payload.partition(":")
|
||||||
|
if not separator:
|
||||||
|
continue
|
||||||
|
service = service.strip().strip('"')
|
||||||
|
package_name = service.split("/")[0]
|
||||||
|
elif line.find("-job") > 0:
|
||||||
|
event = "end_job"
|
||||||
|
payload = line.split("-job=", 1)[1]
|
||||||
|
uid, separator, service = payload.partition(":")
|
||||||
|
if not separator:
|
||||||
|
continue
|
||||||
|
service = service.strip().strip('"')
|
||||||
|
package_name = service.split("/")[0]
|
||||||
|
elif line.find("+running +wake_lock=") > 0:
|
||||||
|
payload = line.split("+running +wake_lock=", 1)[1]
|
||||||
|
uid, separator, _ = payload.partition(":")
|
||||||
|
if not separator:
|
||||||
|
continue
|
||||||
|
event = "wake"
|
||||||
|
service = (
|
||||||
|
line[line.find("*walarm*:") + 9 :].split(" ")[0].strip('"').strip()
|
||||||
|
)
|
||||||
|
if service == "" or "/" not in service:
|
||||||
|
continue
|
||||||
|
|
||||||
|
package_name = service.split("/")[0]
|
||||||
|
elif (line.find("+top=") > 0) or (line.find("-top") > 0):
|
||||||
|
if line.find("+top=") > 0:
|
||||||
|
event = "start_top"
|
||||||
|
top_pos = line.find("+top=")
|
||||||
|
else:
|
||||||
|
event = "end_top"
|
||||||
|
top_pos = line.find("-top=")
|
||||||
|
colon_pos = top_pos + line[top_pos:].find(":")
|
||||||
|
uid = line[top_pos + 5 : colon_pos]
|
||||||
|
service = ""
|
||||||
|
package_name = line[colon_pos + 1 :].strip('"')
|
||||||
else:
|
else:
|
||||||
time_elapsed = fields[0]
|
continue
|
||||||
elapsed = self._elapsed_seconds(time_elapsed)
|
|
||||||
line_time = None
|
|
||||||
|
|
||||||
timestamp = line_time
|
self.results.append(
|
||||||
if timestamp is None and anchor_time is not None and elapsed is not None:
|
{
|
||||||
timestamp = anchor_time + datetime.timedelta(
|
"time_elapsed": time_elapsed,
|
||||||
seconds=elapsed - anchor_elapsed
|
"event": event,
|
||||||
)
|
"uid": uid,
|
||||||
|
"package_name": package_name,
|
||||||
def add(
|
"service": service,
|
||||||
event: str, uid: str, service: str, package_name: str | None
|
}
|
||||||
) -> None:
|
)
|
||||||
self.results.append(
|
|
||||||
{
|
|
||||||
"time_elapsed": time_elapsed,
|
|
||||||
"timestamp": convert_datetime_to_iso(timestamp)
|
|
||||||
if timestamp
|
|
||||||
else None,
|
|
||||||
"event": event,
|
|
||||||
"uid": uid,
|
|
||||||
"package_name": package_name,
|
|
||||||
"service": service,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
for sign, uid, raw_service in re.findall(
|
|
||||||
r"([+-])job=([^:\s]+):\"([^\"]+)\"", line
|
|
||||||
):
|
|
||||||
service = self._normalize_service(raw_service)
|
|
||||||
add(
|
|
||||||
"start_job" if sign == "+" else "end_job",
|
|
||||||
uid,
|
|
||||||
service,
|
|
||||||
self._package_from_name(service) or service.split("/", 1)[0],
|
|
||||||
)
|
|
||||||
|
|
||||||
for sign, uid, package_name in re.findall(
|
|
||||||
r"([+-])top=([^:\s]+):\"([^\"]+)\"", line
|
|
||||||
):
|
|
||||||
add(
|
|
||||||
"start_top" if sign == "+" else "end_top",
|
|
||||||
uid,
|
|
||||||
"",
|
|
||||||
package_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
wake_match = re.search(r"\+wake_lock=([^:\s]+):\"([^\"]+)\"", line)
|
|
||||||
if wake_match:
|
|
||||||
wake_name = wake_match.group(2)
|
|
||||||
add(
|
|
||||||
"wake",
|
|
||||||
wake_match.group(1),
|
|
||||||
wake_name,
|
|
||||||
self._package_from_name(wake_name),
|
|
||||||
)
|
|
||||||
@@ -29,27 +29,16 @@ class DumpsysDBInfoArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
def parse(self, output: str) -> None:
|
def parse(self, output: str) -> None:
|
||||||
rxp = re.compile(
|
rxp = re.compile(
|
||||||
r"^\s*\d+:\s*\[((?:\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\]\s*"
|
r".*\[((?:[0-9]{4}-)?[0-9]{2}-[0-9]{2} "
|
||||||
r"(?:\[Pid:\((\d+)\)\])?\s*([\w-]+) took (\d+)ms - ([^,]+),"
|
r"[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\]\s*"
|
||||||
r"\s*sql=\"(.*)\"(?:, path=(.*))?$"
|
r"(?:\[Pid:\((\d+)\)\])?([\w-]+).*?sql=\"(.+?)\""
|
||||||
)
|
) # pylint: disable=line-too-long
|
||||||
|
|
||||||
pool: str | None = None
|
pool = None
|
||||||
connection_number: int | None = None
|
|
||||||
is_primary: bool | None = None
|
|
||||||
in_operations = False
|
in_operations = False
|
||||||
for line in output.splitlines():
|
for line in output.splitlines():
|
||||||
if line.startswith("Connection pool for "):
|
if line.startswith("Connection pool for "):
|
||||||
pool = line.replace("Connection pool for ", "").rstrip(":")
|
pool = line.replace("Connection pool for ", "").rstrip(":")
|
||||||
in_operations = False
|
|
||||||
|
|
||||||
connection_match = re.match(r"\s+Connection #(\d+):", line)
|
|
||||||
if connection_match:
|
|
||||||
connection_number = int(connection_match.group(1))
|
|
||||||
is_primary = None
|
|
||||||
|
|
||||||
if line.strip().startswith("isPrimaryConnection:"):
|
|
||||||
is_primary = line.strip().split(":", 1)[1].strip() == "true"
|
|
||||||
|
|
||||||
if not pool:
|
if not pool:
|
||||||
continue
|
continue
|
||||||
@@ -63,6 +52,7 @@ class DumpsysDBInfoArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
if not line.startswith(" "):
|
if not line.startswith(" "):
|
||||||
in_operations = False
|
in_operations = False
|
||||||
|
pool = None
|
||||||
continue
|
continue
|
||||||
|
|
||||||
match = rxp.match(line)
|
match = rxp.match(line)
|
||||||
@@ -70,15 +60,11 @@ class DumpsysDBInfoArtifact(AndroidArtifact):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"timestamp": match.group(1),
|
"isodate": match.group(1),
|
||||||
"pid": int(match.group(2)) if match.group(2) else None,
|
|
||||||
"action": match.group(3),
|
"action": match.group(3),
|
||||||
"duration_ms": int(match.group(4)),
|
"sql": match.group(4),
|
||||||
"status": match.group(5),
|
"path": pool,
|
||||||
"sql": match.group(6),
|
|
||||||
"path": match.group(7) or pool,
|
|
||||||
"pool_path": pool,
|
|
||||||
"connection_number": connection_number,
|
|
||||||
"is_primary": is_primary,
|
|
||||||
}
|
}
|
||||||
|
if match.group(2):
|
||||||
|
result["pid"] = match.group(2)
|
||||||
self.results.append(result)
|
self.results.append(result)
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
from .package_resolvers import parse_resolver_table
|
|
||||||
|
|
||||||
|
|
||||||
class DumpsysPackageActivitiesArtifact(AndroidArtifact):
|
class DumpsysPackageActivitiesArtifact(AndroidArtifact):
|
||||||
@@ -20,11 +19,67 @@ class DumpsysPackageActivitiesArtifact(AndroidArtifact):
|
|||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
def parse(self, content: str) -> None:
|
def parse(self, content: str):
|
||||||
"""
|
"""
|
||||||
Parse the Dumpsys Package section for activities
|
Parse the Dumpsys Package section for activities
|
||||||
Adds results to self.results
|
Adds results to self.results
|
||||||
|
|
||||||
:param content: content of the package section (string)
|
:param content: content of the package section (string)
|
||||||
"""
|
"""
|
||||||
self.results = parse_resolver_table(content, "Activity")
|
self.results = []
|
||||||
|
|
||||||
|
in_activity_resolver_table = False
|
||||||
|
in_non_data_actions = False
|
||||||
|
intent = None
|
||||||
|
for line in content.splitlines():
|
||||||
|
if line.startswith("Activity Resolver Table:"):
|
||||||
|
in_activity_resolver_table = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not in_activity_resolver_table:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if line.startswith(" Non-Data Actions:"):
|
||||||
|
in_non_data_actions = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not in_non_data_actions:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we hit an empty line, the Non-Data Actions section should be
|
||||||
|
# finished.
|
||||||
|
if line.strip() == "":
|
||||||
|
break
|
||||||
|
|
||||||
|
# We detect the action name.
|
||||||
|
if (
|
||||||
|
line.startswith(" " * 6)
|
||||||
|
and not line.startswith(" " * 8)
|
||||||
|
and ":" in line
|
||||||
|
):
|
||||||
|
intent = line.strip().replace(":", "")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we are not in an intent block yet, skip.
|
||||||
|
if not intent:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we are in a block but the line does not start with 8 spaces
|
||||||
|
# it means the block ended a new one started, so we reset and
|
||||||
|
# continue.
|
||||||
|
if not line.startswith(" " * 8):
|
||||||
|
intent = None
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we got this far, we are processing receivers for the
|
||||||
|
# activities we are interested in.
|
||||||
|
activity = line.strip().split(" ")[1]
|
||||||
|
package_name = activity.split("/")[0]
|
||||||
|
|
||||||
|
self.results.append(
|
||||||
|
{
|
||||||
|
"intent": intent,
|
||||||
|
"package_name": package_name,
|
||||||
|
"activity": activity,
|
||||||
|
}
|
||||||
|
)
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
# Mobile Verification Toolkit (MVT)
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from typing import Any
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from mvt.android.utils import ROOT_PACKAGES
|
from mvt.android.utils import ROOT_PACKAGES
|
||||||
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
||||||
@@ -12,244 +12,222 @@ from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
|||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
|
|
||||||
def _value(raw: str) -> Any:
|
|
||||||
if raw == "null":
|
|
||||||
return None
|
|
||||||
if raw in ("true", "false"):
|
|
||||||
return raw == "true"
|
|
||||||
try:
|
|
||||||
return int(raw)
|
|
||||||
except ValueError:
|
|
||||||
return raw
|
|
||||||
|
|
||||||
|
|
||||||
class DumpsysPackagesArtifact(AndroidArtifact):
|
class DumpsysPackagesArtifact(AndroidArtifact):
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
alerted_root_packages = set()
|
alerted_root_packages = set()
|
||||||
for result in self.results:
|
for result in self.results:
|
||||||
package_name = result["package_name"]
|
if result["package_name"] in ROOT_PACKAGES:
|
||||||
if (
|
if result["package_name"] in alerted_root_packages:
|
||||||
package_name in ROOT_PACKAGES
|
continue
|
||||||
and package_name not in alerted_root_packages
|
alerted_root_packages.add(result["package_name"])
|
||||||
):
|
|
||||||
alerted_root_packages.add(package_name)
|
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(
|
||||||
f'Found an installed package related to rooting/jailbreaking: "{package_name}"',
|
f'Found an installed package related to rooting/jailbreaking: "{result["package_name"]}"',
|
||||||
"",
|
"",
|
||||||
result,
|
result,
|
||||||
)
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
continue
|
continue
|
||||||
ioc_match = self.indicators.check_app_id(package_name)
|
|
||||||
|
ioc_match = self.indicators.check_app_id(result.get("package_name", ""))
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
)
|
)
|
||||||
|
|
||||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||||
|
records = []
|
||||||
timestamps = [
|
timestamps = [
|
||||||
("package_install", record.get("timestamp")),
|
{"event": "package_install", "timestamp": record["timestamp"]},
|
||||||
("package_last_update", record.get("last_update_time")),
|
|
||||||
]
|
|
||||||
timestamps.extend(
|
|
||||||
("package_first_install", user.get("first_install_time"))
|
|
||||||
for user in record.get("users", [])
|
|
||||||
)
|
|
||||||
return [
|
|
||||||
{
|
{
|
||||||
"timestamp": timestamp,
|
"event": "package_first_install",
|
||||||
"module": self.__class__.__name__,
|
"timestamp": record["first_install_time"],
|
||||||
"event": event,
|
},
|
||||||
"data": f"Install or update of package {record['package_name']}",
|
{"event": "package_last_update", "timestamp": record["last_update_time"]},
|
||||||
}
|
|
||||||
for event, timestamp in timestamps
|
|
||||||
if timestamp
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
for timestamp in timestamps:
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": timestamp["timestamp"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": timestamp["event"],
|
||||||
|
"data": f"Install or update of package {record['package_name']}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return records
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _permission(line: str, permission_type: str) -> dict:
|
def parse_dumpsys_package_for_details(output: str) -> Dict[str, Any]:
|
||||||
name, _, details = line.strip().partition(":")
|
"""
|
||||||
granted_match = re.search(r"granted=(true|false)", details)
|
Parse one entry of a dumpsys package information
|
||||||
flags_match = re.search(r"flags=\[\s*([^]]*)\]", details)
|
"""
|
||||||
return {
|
details: Dict[str, Any] = {
|
||||||
"name": name,
|
"uid": "",
|
||||||
"type": permission_type,
|
"version_name": "",
|
||||||
"granted": granted_match.group(1) == "true" if granted_match else None,
|
"version_code": "",
|
||||||
"flags": [
|
"timestamp": "",
|
||||||
flag.strip()
|
"first_install_time": "",
|
||||||
for flag in (flags_match.group(1).split("|") if flags_match else [])
|
"last_update_time": "",
|
||||||
if flag.strip()
|
"installer": "",
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def parse_dumpsys_package_for_details(cls, output: str) -> dict[str, Any]:
|
|
||||||
details: dict[str, Any] = {
|
|
||||||
"app_id": None,
|
|
||||||
"version_name": None,
|
|
||||||
"version_code": None,
|
|
||||||
"min_sdk": None,
|
|
||||||
"target_sdk": None,
|
|
||||||
"timestamp": None,
|
|
||||||
"last_update_time": None,
|
|
||||||
"installer": None,
|
|
||||||
"system": False,
|
"system": False,
|
||||||
"permissions": [],
|
"permissions": list(),
|
||||||
"requested_permissions": [],
|
"requested_permissions": list(),
|
||||||
"users": [],
|
|
||||||
}
|
}
|
||||||
permission_section: str | None = None
|
in_install_permissions = False
|
||||||
current_user: dict[str, Any] | None = None
|
in_runtime_permissions = False
|
||||||
legacy_first_install: str | None = None
|
in_declared_permissions = False
|
||||||
|
in_requested_permissions = True
|
||||||
|
current_user: Optional[int] = None
|
||||||
|
first_install_times: Dict[Optional[int], str] = {}
|
||||||
|
runtime_permissions: Dict[Optional[int], List[Dict[str, Any]]] = {}
|
||||||
for line in output.splitlines():
|
for line in output.splitlines():
|
||||||
stripped = line.strip()
|
user_match = re.match(r"User (\d+):", line.strip())
|
||||||
user_match = re.match(r"User (\d+):\s*(.*)", stripped)
|
|
||||||
if user_match:
|
if user_match:
|
||||||
current_user = {"user_id": int(user_match.group(1)), "permissions": []}
|
current_user = int(user_match.group(1))
|
||||||
for key, raw in re.findall(r"(\w+)=([^\s]+)", user_match.group(2)):
|
|
||||||
clean_key = {
|
|
||||||
"notLaunched": "not_launched",
|
|
||||||
"installReason": "install_reason",
|
|
||||||
"uninstallReason": "uninstall_reason",
|
|
||||||
"dataDir": "data_dir",
|
|
||||||
}.get(key, re.sub(r"(?<!^)(?=[A-Z])", "_", key).lower())
|
|
||||||
current_user[clean_key] = _value(raw)
|
|
||||||
details["users"].append(current_user)
|
|
||||||
permission_section = None
|
|
||||||
continue
|
|
||||||
|
|
||||||
header = stripped.lower()
|
if in_install_permissions:
|
||||||
if header in {
|
if line.startswith(" " * 4) and not line.startswith(" " * 6):
|
||||||
"declared permissions:",
|
in_install_permissions = False
|
||||||
"install permissions:",
|
else:
|
||||||
"requested permissions:",
|
lineinfo = line.strip().split(":")
|
||||||
"runtime permissions:",
|
permission = lineinfo[0]
|
||||||
}:
|
granted = None
|
||||||
permission_section = header.split()[0]
|
if "granted=" in lineinfo[1]:
|
||||||
continue
|
granted = "granted=true" in lineinfo[1]
|
||||||
|
|
||||||
if current_user is not None:
|
details["permissions"].append(
|
||||||
user_property = re.match(
|
{"name": permission, "granted": granted, "type": "install"}
|
||||||
r"(installReason|uninstallReason|dataDir|firstInstallTime)=(.*)",
|
)
|
||||||
stripped,
|
if in_runtime_permissions:
|
||||||
)
|
if not line.startswith(" " * 8):
|
||||||
if user_property:
|
in_runtime_permissions = False
|
||||||
key = {
|
else:
|
||||||
"installReason": "install_reason",
|
lineinfo = line.strip().split(":")
|
||||||
"uninstallReason": "uninstall_reason",
|
permission = lineinfo[0]
|
||||||
"dataDir": "data_dir",
|
granted = None
|
||||||
"firstInstallTime": "first_install_time",
|
if "granted=" in lineinfo[1]:
|
||||||
}[user_property.group(1)]
|
granted = "granted=true" in lineinfo[1]
|
||||||
current_user[key] = _value(user_property.group(2))
|
|
||||||
continue
|
|
||||||
|
|
||||||
if permission_section == "requested" and line.startswith(" "):
|
runtime_permissions.setdefault(current_user, []).append(
|
||||||
details["requested_permissions"].append(stripped)
|
{"name": permission, "granted": granted, "type": "runtime"}
|
||||||
continue
|
)
|
||||||
if permission_section in ("declared", "install") and line.startswith(
|
if in_declared_permissions:
|
||||||
" "
|
if not line.startswith(" " * 6):
|
||||||
):
|
in_declared_permissions = False
|
||||||
details["permissions"].append(
|
else:
|
||||||
cls._permission(stripped, permission_section)
|
permission = line.strip().split(":")[0]
|
||||||
)
|
details["permissions"].append(
|
||||||
continue
|
{"name": permission, "type": "declared"}
|
||||||
if (
|
)
|
||||||
permission_section == "runtime"
|
if in_requested_permissions:
|
||||||
and line.startswith(" ")
|
if not line.startswith(" " * 6):
|
||||||
and current_user is not None
|
in_requested_permissions = False
|
||||||
):
|
else:
|
||||||
current_user["permissions"].append(cls._permission(stripped, "runtime"))
|
details["requested_permissions"].append(line.strip())
|
||||||
continue
|
if line.strip().startswith("userId="):
|
||||||
|
details["uid"] = line.split("=")[1].strip()
|
||||||
|
elif line.strip().startswith("versionName="):
|
||||||
|
details["version_name"] = line.split("=")[1].strip()
|
||||||
|
elif line.strip().startswith("versionCode="):
|
||||||
|
details["version_code"] = line.split("=", 1)[1].strip()
|
||||||
|
elif line.strip().startswith("timeStamp="):
|
||||||
|
details["timestamp"] = line.split("=")[1].strip()
|
||||||
|
elif line.strip().startswith("installerPackageName="):
|
||||||
|
details["installer"] = line.split("=", 1)[1].strip()
|
||||||
|
elif line.strip().startswith("pkgFlags="):
|
||||||
|
details["system"] = "SYSTEM" in line.split("=", 1)[1].split()
|
||||||
|
elif line.strip().startswith("firstInstallTime="):
|
||||||
|
first_install_times[current_user] = line.split("=", 1)[1].strip()
|
||||||
|
elif line.strip().startswith("lastUpdateTime="):
|
||||||
|
details["last_update_time"] = line.split("=")[1].strip()
|
||||||
|
elif line.strip() == "install permissions:":
|
||||||
|
in_install_permissions = True
|
||||||
|
elif line.strip() == "runtime permissions:":
|
||||||
|
in_runtime_permissions = True
|
||||||
|
elif line.strip() == "declared permissions:":
|
||||||
|
in_declared_permissions = True
|
||||||
|
elif line.strip() == "requested permissions:":
|
||||||
|
in_requested_permissions = True
|
||||||
|
|
||||||
simple_match = re.match(
|
if 0 in first_install_times:
|
||||||
r"(appId|userId|versionName|timeStamp|lastUpdateTime|installerPackageName)=(.*)",
|
details["first_install_time"] = first_install_times[0]
|
||||||
stripped,
|
elif None in first_install_times:
|
||||||
)
|
details["first_install_time"] = first_install_times[None]
|
||||||
if simple_match:
|
elif first_install_times:
|
||||||
key = {
|
details["first_install_time"] = next(iter(first_install_times.values()))
|
||||||
"appId": "app_id",
|
|
||||||
"userId": "app_id",
|
|
||||||
"versionName": "version_name",
|
|
||||||
"timeStamp": "timestamp",
|
|
||||||
"lastUpdateTime": "last_update_time",
|
|
||||||
"installerPackageName": "installer",
|
|
||||||
}[simple_match.group(1)]
|
|
||||||
raw_value = simple_match.group(2)
|
|
||||||
details[key] = (
|
|
||||||
_value(raw_value)
|
|
||||||
if key == "app_id" or raw_value == "null"
|
|
||||||
else raw_value
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
if stripped.startswith("pkgFlags="):
|
|
||||||
details["system"] = "SYSTEM" in stripped.split("=", 1)[1].split()
|
|
||||||
continue
|
|
||||||
version_match = re.match(
|
|
||||||
r"versionCode=([^\s]+)(?:\s+minSdk=([^\s]+))?(?:\s+targetSdk=([^\s]+))?",
|
|
||||||
stripped,
|
|
||||||
)
|
|
||||||
if version_match:
|
|
||||||
details["version_code"] = _value(version_match.group(1))
|
|
||||||
details["min_sdk"] = (
|
|
||||||
_value(version_match.group(2)) if version_match.group(2) else None
|
|
||||||
)
|
|
||||||
details["target_sdk"] = (
|
|
||||||
_value(version_match.group(3)) if version_match.group(3) else None
|
|
||||||
)
|
|
||||||
elif stripped.startswith("firstInstallTime="):
|
|
||||||
legacy_first_install = stripped.split("=", 1)[1]
|
|
||||||
|
|
||||||
if legacy_first_install:
|
if 0 in runtime_permissions:
|
||||||
user_zero = next(
|
details["permissions"].extend(runtime_permissions[0])
|
||||||
(user for user in details["users"] if user["user_id"] == 0), None
|
elif None in runtime_permissions:
|
||||||
)
|
details["permissions"].extend(runtime_permissions[None])
|
||||||
if user_zero is None:
|
elif runtime_permissions:
|
||||||
user_zero = {"user_id": 0, "permissions": []}
|
details["permissions"].extend(next(iter(runtime_permissions.values())))
|
||||||
details["users"].append(user_zero)
|
|
||||||
user_zero.setdefault("first_install_time", legacy_first_install)
|
|
||||||
return details
|
return details
|
||||||
|
|
||||||
def parse(self, content: str) -> None:
|
def parse_dumpsys_packages(self, output: str) -> List[Dict[str, Any]]:
|
||||||
self.results: list[dict[str, Any]] = []
|
"""
|
||||||
category: str | None = None
|
Parse the dumpsys package service data
|
||||||
package: dict[str, Any] | None = None
|
"""
|
||||||
block: list[str] = []
|
pkg_rxp = re.compile(r" Package \[(.+?)\].*")
|
||||||
|
|
||||||
def finish() -> None:
|
results = []
|
||||||
nonlocal package, block
|
package_name = None
|
||||||
if package is not None:
|
package = {}
|
||||||
package.update(self.parse_dumpsys_package_for_details("\n".join(block)))
|
lines: list[str] = []
|
||||||
self.results.append(package)
|
for line in output.splitlines():
|
||||||
package = None
|
if line.startswith(" Package ["):
|
||||||
block = []
|
if len(lines) > 0:
|
||||||
|
details = self.parse_dumpsys_package_for_details("\n".join(lines))
|
||||||
|
package.update(details)
|
||||||
|
results.append(package)
|
||||||
|
lines = []
|
||||||
|
package = {}
|
||||||
|
|
||||||
|
matches = pkg_rxp.findall(line)
|
||||||
|
if not matches:
|
||||||
|
continue
|
||||||
|
|
||||||
|
package_name = matches[0]
|
||||||
|
package["package_name"] = package_name
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not package_name:
|
||||||
|
continue
|
||||||
|
|
||||||
|
lines.append(line)
|
||||||
|
|
||||||
|
if len(lines) > 0:
|
||||||
|
details = self.parse_dumpsys_package_for_details("\n".join(lines))
|
||||||
|
package.update(details)
|
||||||
|
results.append(package)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
def parse(self, content: str):
|
||||||
|
"""
|
||||||
|
Parse the Dumpsys Package section for activities
|
||||||
|
Adds results to self.results
|
||||||
|
|
||||||
|
:param content: content of the package section (string)
|
||||||
|
"""
|
||||||
|
self.results = []
|
||||||
|
package = []
|
||||||
|
|
||||||
|
in_package_list = False
|
||||||
for line in content.splitlines():
|
for line in content.splitlines():
|
||||||
if line == "Packages:":
|
if line.startswith("Packages:"):
|
||||||
finish()
|
in_package_list = True
|
||||||
category = "active"
|
|
||||||
continue
|
continue
|
||||||
if line == "Hidden system packages:":
|
|
||||||
finish()
|
if not in_package_list:
|
||||||
category = "hidden_system"
|
|
||||||
continue
|
continue
|
||||||
package_match = re.match(r"^ Package \[([^]]+)\]", line)
|
|
||||||
if package_match and category:
|
if line.strip() == "":
|
||||||
finish()
|
break
|
||||||
package = {
|
|
||||||
"package_name": package_match.group(1),
|
package.append(line)
|
||||||
"package_type": category,
|
|
||||||
}
|
self.results = self.parse_dumpsys_packages("\n".join(package))
|
||||||
continue
|
|
||||||
if (
|
|
||||||
category
|
|
||||||
and line
|
|
||||||
and not line.startswith(" ")
|
|
||||||
and not line.endswith(" overlay paths:")
|
|
||||||
):
|
|
||||||
finish()
|
|
||||||
category = None
|
|
||||||
continue
|
|
||||||
if package is not None:
|
|
||||||
block.append(line)
|
|
||||||
finish()
|
|
||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
|
|
||||||
@@ -27,37 +25,19 @@ class DumpsysPlatformCompatArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
def parse(self, data: str) -> None:
|
def parse(self, data: str) -> None:
|
||||||
for line in data.splitlines():
|
for line in data.splitlines():
|
||||||
match = re.match(r"ChangeId\((\d+);\s*(.*)\)$", line.strip())
|
if not line.startswith("ChangeId(168419799; name=DOWNSCALED;"):
|
||||||
if not match or "rawOverrides={" not in line:
|
|
||||||
continue
|
continue
|
||||||
body = match.group(2)
|
|
||||||
name_match = re.search(r"(?:^|;\s*)name=([^;]+)", body)
|
if line.strip() == "":
|
||||||
state = (
|
break
|
||||||
"enabled"
|
|
||||||
if re.search(r"(?:^|;\s*)enabled(?:;|$)", body)
|
# Look for rawOverrides field
|
||||||
else "disabled"
|
if "rawOverrides={" in line:
|
||||||
)
|
# Extract the content inside the braces for rawOverrides
|
||||||
overridable = bool(re.search(r"(?:^|;\s*)overridable(?:;|$)", body))
|
overrides_field = line.split("rawOverrides={", 1)[1].split("};", 1)[0]
|
||||||
overrides_field = body.split("rawOverrides={", 1)[1].split("}", 1)[0]
|
|
||||||
for entry in overrides_field.split(","):
|
for entry in overrides_field.split(", "):
|
||||||
package_name, separator, raw_value = entry.strip().partition("=")
|
# Extract app name
|
||||||
if not separator:
|
uninstall_app = entry.split("=")[0].strip()
|
||||||
continue
|
|
||||||
value: bool | int | str
|
self.results.append({"package_name": uninstall_app})
|
||||||
if raw_value in ("true", "false"):
|
|
||||||
value = raw_value == "true"
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
value = int(raw_value)
|
|
||||||
except ValueError:
|
|
||||||
value = raw_value
|
|
||||||
self.results.append(
|
|
||||||
{
|
|
||||||
"change_id": int(match.group(1)),
|
|
||||||
"change_name": name_match.group(1) if name_match else None,
|
|
||||||
"change_state": state,
|
|
||||||
"overridable": overridable,
|
|
||||||
"package_name": package_name,
|
|
||||||
"override_value": value,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
@@ -4,7 +4,6 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
from .package_resolvers import parse_resolver_table
|
|
||||||
|
|
||||||
INTENT_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
|
INTENT_NEW_OUTGOING_SMS = "android.provider.Telephony.NEW_OUTGOING_SMS"
|
||||||
INTENT_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
|
INTENT_SMS_RECEIVED = "android.provider.Telephony.SMS_RECEIVED"
|
||||||
@@ -19,45 +18,115 @@ class DumpsysReceiversArtifact(AndroidArtifact):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
for receiver in self.results:
|
for intent, receivers in self.results.items():
|
||||||
intent = receiver["key"]
|
for receiver in receivers:
|
||||||
if intent == INTENT_NEW_OUTGOING_SMS:
|
if intent == INTENT_NEW_OUTGOING_SMS:
|
||||||
self.log.info(
|
self.log.info(
|
||||||
'Found a receiver to intercept outgoing SMS messages: "%s"',
|
'Found a receiver to intercept outgoing SMS messages: "%s"',
|
||||||
receiver["component"],
|
receiver["receiver"],
|
||||||
)
|
)
|
||||||
elif intent == INTENT_SMS_RECEIVED:
|
elif intent == INTENT_SMS_RECEIVED:
|
||||||
self.log.info(
|
self.log.info(
|
||||||
'Found a receiver to intercept incoming SMS messages: "%s"',
|
'Found a receiver to intercept incoming SMS messages: "%s"',
|
||||||
receiver["component"],
|
receiver["receiver"],
|
||||||
)
|
)
|
||||||
elif intent == INTENT_DATA_SMS_RECEIVED:
|
elif intent == INTENT_DATA_SMS_RECEIVED:
|
||||||
self.log.info(
|
self.log.info(
|
||||||
'Found a receiver to intercept incoming data SMS message: "%s"',
|
'Found a receiver to intercept incoming data SMS message: "%s"',
|
||||||
receiver["component"],
|
receiver["receiver"],
|
||||||
)
|
)
|
||||||
elif intent == INTENT_PHONE_STATE:
|
elif intent == INTENT_PHONE_STATE:
|
||||||
self.log.info(
|
self.log.info(
|
||||||
'Found a receiver monitoring telephony state/incoming calls: "%s"',
|
"Found a receiver monitoring "
|
||||||
receiver["component"],
|
'telephony state/incoming calls: "%s"',
|
||||||
)
|
receiver["receiver"],
|
||||||
elif intent == INTENT_NEW_OUTGOING_CALL:
|
)
|
||||||
self.log.info(
|
elif intent == INTENT_NEW_OUTGOING_CALL:
|
||||||
'Found a receiver monitoring outgoing calls: "%s"',
|
self.log.info(
|
||||||
receiver["component"],
|
'Found a receiver monitoring outgoing calls: "%s"',
|
||||||
)
|
receiver["receiver"],
|
||||||
|
)
|
||||||
|
|
||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
ioc_match = self.indicators.check_app_id(receiver["package_name"])
|
ioc_match = self.indicators.check_app_id(receiver["package_name"])
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message,
|
ioc_match.message,
|
||||||
"",
|
"",
|
||||||
receiver,
|
{intent: receiver},
|
||||||
matched_indicator=ioc_match.ioc,
|
matched_indicator=ioc_match.ioc,
|
||||||
)
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
def parse(self, output: str) -> None:
|
def parse(self, output: str) -> None:
|
||||||
self.results = parse_resolver_table(output, "Receiver")
|
self.results: dict[str, list[dict[str, str]]] = {}
|
||||||
|
|
||||||
|
in_receiver_resolver_table = False
|
||||||
|
in_non_data_actions = False
|
||||||
|
intent = None
|
||||||
|
for line in output.splitlines():
|
||||||
|
if line.startswith("Receiver Resolver Table:"):
|
||||||
|
in_receiver_resolver_table = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not in_receiver_resolver_table:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if line.startswith(" Non-Data Actions:"):
|
||||||
|
in_non_data_actions = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not in_non_data_actions:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we hit an empty line, the Non-Data Actions section should be
|
||||||
|
# finished.
|
||||||
|
if line.strip() == "":
|
||||||
|
break
|
||||||
|
|
||||||
|
# We detect the action name.
|
||||||
|
if (
|
||||||
|
line.startswith(" " * 6)
|
||||||
|
and not line.startswith(" " * 8)
|
||||||
|
and ":" in line
|
||||||
|
):
|
||||||
|
intent = line.strip().replace(":", "")
|
||||||
|
self.results[intent] = []
|
||||||
|
continue
|
||||||
|
|
||||||
|
parts = line.strip().split(" ")
|
||||||
|
if len(parts) < 2:
|
||||||
|
# A single-token line here is not a receiver. Real dumpstate
|
||||||
|
# output can print an action header mis-indented (observed with
|
||||||
|
# 15 leading spaces instead of 6), which used to raise
|
||||||
|
# IndexError and abort the whole module. Treat a trailing-colon
|
||||||
|
# token as the next action, skip anything else.
|
||||||
|
if parts[0].endswith(":"):
|
||||||
|
intent = parts[0][:-1]
|
||||||
|
self.results.setdefault(intent, [])
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we are not in an intent block yet, skip.
|
||||||
|
if not intent:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we are in a block but the line does not start with 8 spaces
|
||||||
|
# it means the block ended a new one started, so we reset and
|
||||||
|
# continue.
|
||||||
|
if not line.startswith(" " * 8):
|
||||||
|
intent = None
|
||||||
|
continue
|
||||||
|
|
||||||
|
# If we got this far, we are processing receivers for the
|
||||||
|
# activities we are interested in.
|
||||||
|
receiver = parts[1]
|
||||||
|
package_name = receiver.split("/")[0]
|
||||||
|
|
||||||
|
self.results[intent].append(
|
||||||
|
{
|
||||||
|
"package_name": package_name,
|
||||||
|
"receiver": receiver,
|
||||||
|
}
|
||||||
|
)
|
||||||
@@ -28,16 +28,19 @@ INTERESTING_PROPERTIES = [
|
|||||||
class GetProp(AndroidArtifact):
|
class GetProp(AndroidArtifact):
|
||||||
def parse(self, entry: str) -> None:
|
def parse(self, entry: str) -> None:
|
||||||
self.results: List[Dict[str, str]] = []
|
self.results: List[Dict[str, str]] = []
|
||||||
# A property value may span several lines: persist.sys.boot.reason.history
|
rxp = re.compile(r"\[(.+?)\]: \[(.+?)\]")
|
||||||
# prints one boot per line. Matching the whole section instead of line by
|
|
||||||
# line lets a value run to the first closing bracket that ends a line.
|
|
||||||
rxp = re.compile(
|
|
||||||
r"^[ \t]*\[([^]]+)\]: \[(.*?)\][ \t\r]*$",
|
|
||||||
re.MULTILINE | re.DOTALL,
|
|
||||||
)
|
|
||||||
|
|
||||||
for name, value in rxp.findall(entry):
|
for line in entry.splitlines():
|
||||||
self.results.append({"name": name, "value": value})
|
line = line.strip()
|
||||||
|
if line == "":
|
||||||
|
continue
|
||||||
|
|
||||||
|
matches = re.findall(rxp, line)
|
||||||
|
if not matches or len(matches[0]) != 2:
|
||||||
|
continue
|
||||||
|
|
||||||
|
prop_entry = {"name": matches[0][0], "value": matches[0][1]}
|
||||||
|
self.results.append(prop_entry)
|
||||||
|
|
||||||
def get_device_timezone(self) -> str | None:
|
def get_device_timezone(self) -> str | None:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -117,51 +117,6 @@ class Mounts(AndroidArtifact):
|
|||||||
# Skip lines that don't match expected format
|
# Skip lines that don't match expected format
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def parse_mountinfo(entry: str, process_id: int) -> list[dict[str, Any]]:
|
|
||||||
"""Parse Linux /proc/PID/mountinfo records."""
|
|
||||||
results = []
|
|
||||||
for line in entry.splitlines():
|
|
||||||
fields = line.split()
|
|
||||||
if "-" not in fields:
|
|
||||||
continue
|
|
||||||
separator = fields.index("-")
|
|
||||||
if separator < 6 or len(fields) < separator + 4:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
mount_id = int(fields[0])
|
|
||||||
parent_id = int(fields[1])
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
mount_options = fields[5].split(",")
|
|
||||||
super_options = fields[separator + 3].split(",")
|
|
||||||
options = list(dict.fromkeys(mount_options + super_options))
|
|
||||||
mount_point = fields[4].replace("\\040", " ")
|
|
||||||
device = fields[separator + 2].replace("\\040", " ")
|
|
||||||
filesystem_type = fields[separator + 1]
|
|
||||||
is_system = mount_point in SUSPICIOUS_MOUNT_POINTS or any(
|
|
||||||
mount_point.startswith(f"{prefix}/")
|
|
||||||
for prefix in SUSPICIOUS_MOUNT_POINTS
|
|
||||||
)
|
|
||||||
results.append(
|
|
||||||
{
|
|
||||||
"mount_id": mount_id,
|
|
||||||
"parent_id": parent_id,
|
|
||||||
"major_minor": fields[2],
|
|
||||||
"root": fields[3].replace("\\040", " "),
|
|
||||||
"mount_point": mount_point,
|
|
||||||
"device": device,
|
|
||||||
"filesystem_type": filesystem_type,
|
|
||||||
"mount_options": ",".join(options),
|
|
||||||
"options_list": options,
|
|
||||||
"optional_fields": fields[6:separator],
|
|
||||||
"is_system_partition": is_system,
|
|
||||||
"is_read_write": "rw" in options,
|
|
||||||
"process_ids": [process_id],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return results
|
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
"""
|
"""
|
||||||
Check for suspicious mount configurations that may indicate root access
|
Check for suspicious mount configurations that may indicate root access
|
||||||
|
|||||||
@@ -1,70 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
|
||||||
RESOLVER_TYPES = {
|
|
||||||
"Full MIME Types": "full_mime_type",
|
|
||||||
"Base MIME Types": "base_mime_type",
|
|
||||||
"Wild MIME Types": "wild_mime_type",
|
|
||||||
"Schemes": "scheme",
|
|
||||||
"Non-Data Actions": "non_data_action",
|
|
||||||
"MIME Typed Actions": "mime_typed_action",
|
|
||||||
}
|
|
||||||
|
|
||||||
ENTRY_RE = re.compile(
|
|
||||||
r"^\s{8,}[0-9a-fA-F]+\s+(?P<component>\S+)"
|
|
||||||
r"(?:\s+\((?P<filters>\d+)\s+filters?\))?\s*$"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_resolver_table(content: str, table_name: str) -> list[dict]:
|
|
||||||
"""Parse every resolver category from a dumpsys package table."""
|
|
||||||
results: list[dict] = []
|
|
||||||
in_table = False
|
|
||||||
resolver_type: str | None = None
|
|
||||||
key: str | None = None
|
|
||||||
|
|
||||||
for line in content.splitlines():
|
|
||||||
if line.startswith(f"{table_name} Resolver Table:"):
|
|
||||||
in_table = True
|
|
||||||
continue
|
|
||||||
if not in_table:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if line and not line.startswith(" "):
|
|
||||||
break
|
|
||||||
|
|
||||||
section_match = re.match(r"^ {2}([^ ].*):\s*$", line)
|
|
||||||
if section_match:
|
|
||||||
resolver_type = RESOLVER_TYPES.get(section_match.group(1))
|
|
||||||
key = None
|
|
||||||
continue
|
|
||||||
if resolver_type is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
key_match = re.match(r"^ {6,}([^ ].*):\s*$", line)
|
|
||||||
if key_match:
|
|
||||||
key = key_match.group(1)
|
|
||||||
continue
|
|
||||||
if key is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
entry_match = ENTRY_RE.match(line)
|
|
||||||
if not entry_match:
|
|
||||||
continue
|
|
||||||
component = entry_match.group("component")
|
|
||||||
results.append(
|
|
||||||
{
|
|
||||||
"resolver_type": resolver_type,
|
|
||||||
"key": key,
|
|
||||||
"package_name": component.split("/", 1)[0],
|
|
||||||
"component": component,
|
|
||||||
"filter_count": int(entry_match.group("filters") or 1),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return results
|
|
||||||
@@ -1,91 +1,72 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
# Mobile Verification Toolkit (MVT)
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
|
|
||||||
FIELD_NAMES = {
|
|
||||||
"LABEL": "label",
|
|
||||||
"USER": "user",
|
|
||||||
"PID": "pid",
|
|
||||||
"TID": "tid",
|
|
||||||
"PPID": "ppid",
|
|
||||||
"VSZ": "virtual_memory_size",
|
|
||||||
"RSS": "resident_set_size",
|
|
||||||
"WCHAN": "wchan",
|
|
||||||
"ADDR": "address",
|
|
||||||
"S": "state",
|
|
||||||
"PRI": "priority",
|
|
||||||
"NI": "nice",
|
|
||||||
"RTPRIO": "realtime_priority",
|
|
||||||
"SCH": "scheduler",
|
|
||||||
"PCY": "policy",
|
|
||||||
"TIME": "cpu_time",
|
|
||||||
"CMD": "command",
|
|
||||||
"NAME": "command",
|
|
||||||
}
|
|
||||||
INTEGER_FIELDS = {
|
|
||||||
"pid",
|
|
||||||
"tid",
|
|
||||||
"ppid",
|
|
||||||
"virtual_memory_size",
|
|
||||||
"resident_set_size",
|
|
||||||
"priority",
|
|
||||||
"nice",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class Processes(AndroidArtifact):
|
class Processes(AndroidArtifact):
|
||||||
def parse(self, entry: str) -> None:
|
def parse(self, entry: str) -> None:
|
||||||
self.results = []
|
for line in entry.splitlines()[1:]:
|
||||||
lines = [line for line in entry.splitlines() if line.strip()]
|
proc = line.split()
|
||||||
if not lines:
|
|
||||||
return
|
|
||||||
headers = lines[0].split()
|
|
||||||
if not all(header in FIELD_NAMES for header in headers):
|
|
||||||
return
|
|
||||||
|
|
||||||
for line in lines[1:]:
|
# Skip empty lines
|
||||||
values = line.split(None, len(headers) - 1)
|
if len(proc) == 0:
|
||||||
if len(values) != len(headers):
|
|
||||||
continue
|
continue
|
||||||
result = {}
|
|
||||||
valid = True
|
# Sometimes WCHAN is empty.
|
||||||
for header, raw in zip(headers, values):
|
if len(proc) == 8:
|
||||||
key = FIELD_NAMES[header]
|
proc = proc[:5] + [""] + proc[5:]
|
||||||
value: str | int = raw.strip("[]") if key == "command" else raw
|
|
||||||
if key in INTEGER_FIELDS:
|
# Sometimes there is the security label.
|
||||||
try:
|
if proc[0].startswith("u:r"):
|
||||||
value = int(value)
|
label = proc[0]
|
||||||
except ValueError:
|
proc = proc[1:]
|
||||||
valid = False
|
else:
|
||||||
break
|
label = ""
|
||||||
result[key] = value
|
|
||||||
if valid:
|
# Sometimes there is no WCHAN.
|
||||||
self.results.append(result)
|
if len(proc) < 9:
|
||||||
|
proc = proc[:5] + [""] + proc[5:]
|
||||||
|
|
||||||
|
self.results.append(
|
||||||
|
{
|
||||||
|
"user": proc[0],
|
||||||
|
"pid": int(proc[1]),
|
||||||
|
"ppid": int(proc[2]),
|
||||||
|
"virtual_memory_size": int(proc[3]),
|
||||||
|
"resident_set_size": int(proc[4]),
|
||||||
|
"wchan": proc[5],
|
||||||
|
"aprocress": proc[6],
|
||||||
|
"stat": proc[7],
|
||||||
|
"proc_name": proc[8].strip("[]"),
|
||||||
|
"label": label,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
for result in self.results:
|
for result in self.results:
|
||||||
command = result.get("command", "")
|
proc_name = result.get("proc_name", "")
|
||||||
if not isinstance(command, str):
|
if not proc_name:
|
||||||
continue
|
continue
|
||||||
process_name = command.rsplit("/", 1)[-1]
|
|
||||||
if not process_name or process_name == "gatekeeperd":
|
# Skipping this process because of false positives.
|
||||||
|
if result["proc_name"] == "gatekeeperd":
|
||||||
continue
|
continue
|
||||||
for checker in (
|
|
||||||
self.indicators.check_app_id,
|
ioc_match = self.indicators.check_app_id(proc_name)
|
||||||
self.indicators.check_process,
|
if ioc_match:
|
||||||
):
|
self.alertstore.critical(
|
||||||
ioc_match = checker(process_name)
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
if ioc_match:
|
)
|
||||||
self.alertstore.critical(
|
continue
|
||||||
ioc_match.message,
|
|
||||||
"",
|
ioc_match = self.indicators.check_process(proc_name)
|
||||||
result,
|
if ioc_match:
|
||||||
matched_indicator=ioc_match.ioc,
|
self.alertstore.critical(
|
||||||
)
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
break
|
)
|
||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from .artifact import AndroidArtifact
|
from .artifact import AndroidArtifact
|
||||||
|
|
||||||
ANDROID_DANGEROUS_SETTINGS = [
|
ANDROID_DANGEROUS_SETTINGS = [
|
||||||
@@ -62,28 +60,6 @@ ANDROID_DANGEROUS_SETTINGS = [
|
|||||||
|
|
||||||
|
|
||||||
class Settings(AndroidArtifact):
|
class Settings(AndroidArtifact):
|
||||||
def parse(self, content: str) -> None:
|
|
||||||
self.results: dict[str, dict[str, str]] = {}
|
|
||||||
namespace: str | None = None
|
|
||||||
for line in content.splitlines():
|
|
||||||
heading = re.match(
|
|
||||||
r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$",
|
|
||||||
line.strip(),
|
|
||||||
)
|
|
||||||
if heading:
|
|
||||||
namespace = f"{heading.group(1).lower()}:user_{heading.group(2)}"
|
|
||||||
self.results[namespace] = {}
|
|
||||||
continue
|
|
||||||
if namespace is None or not line.startswith("_id:"):
|
|
||||||
continue
|
|
||||||
setting = re.match(
|
|
||||||
r"^_id:\S+\s+name:(.*?)\s+pkg:.*?\s+value:(.*?)"
|
|
||||||
r"(?:\s+default:.*\s+defaultSystemSet:(?:true|false))?$",
|
|
||||||
line,
|
|
||||||
)
|
|
||||||
if setting:
|
|
||||||
self.results[namespace][setting.group(1)] = setting.group(2)
|
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
for namespace, settings in self.results.items():
|
for namespace, settings in self.results.items():
|
||||||
for key, value in settings.items():
|
for key, value in settings.items():
|
||||||
|
|||||||
@@ -131,13 +131,6 @@ class TombstoneCrashArtifact(AndroidArtifact):
|
|||||||
self, file_name: str, file_timestamp: datetime.datetime, data: bytes
|
self, file_name: str, file_timestamp: datetime.datetime, data: bytes
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Parse Android tombstone crash files from a protobuf object."""
|
"""Parse Android tombstone crash files from a protobuf object."""
|
||||||
self.results.append(self.parse_protobuf_record(file_name, file_timestamp, data))
|
|
||||||
|
|
||||||
def parse_protobuf_record(
|
|
||||||
self, file_name: str, file_timestamp: datetime.datetime, data: bytes
|
|
||||||
) -> dict:
|
|
||||||
if not data:
|
|
||||||
raise ValueError("empty protobuf tombstone")
|
|
||||||
tombstone_pb = Tombstone().parse(data)
|
tombstone_pb = Tombstone().parse(data)
|
||||||
tombstone_dict = tombstone_pb.to_dict(
|
tombstone_dict = tombstone_pb.to_dict(
|
||||||
casing=betterproto2.Casing.SNAKE, include_default_values=True
|
casing=betterproto2.Casing.SNAKE, include_default_values=True
|
||||||
@@ -150,31 +143,20 @@ class TombstoneCrashArtifact(AndroidArtifact):
|
|||||||
tombstone_dict["file_name"] = file_name
|
tombstone_dict["file_name"] = file_name
|
||||||
tombstone_dict["file_timestamp"] = convert_datetime_to_iso(file_timestamp)
|
tombstone_dict["file_timestamp"] = convert_datetime_to_iso(file_timestamp)
|
||||||
tombstone_dict["process_name"] = self._proccess_name_from_thread(tombstone_dict)
|
tombstone_dict["process_name"] = self._proccess_name_from_thread(tombstone_dict)
|
||||||
if isinstance(tombstone_dict.get("selinux_label"), str):
|
|
||||||
tombstone_dict["selinux_label"] = tombstone_dict["selinux_label"].rstrip(
|
|
||||||
"\x00"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Confirm the tombstone is valid, and matches the output model
|
# Confirm the tombstone is valid, and matches the output model
|
||||||
tombstone = TombstoneCrashResult.model_validate(tombstone_dict)
|
tombstone = TombstoneCrashResult.model_validate(tombstone_dict)
|
||||||
return tombstone.model_dump()
|
self.results.append(tombstone.model_dump())
|
||||||
|
|
||||||
def parse(
|
def parse(
|
||||||
self, file_name: str, file_timestamp: datetime.datetime, content: bytes
|
self, file_name: str, file_timestamp: datetime.datetime, content: bytes
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Parse text Android tombstone crash files."""
|
"""Parse text Android tombstone crash files."""
|
||||||
self.results.append(self.parse_text_record(file_name, file_timestamp, content))
|
|
||||||
|
|
||||||
def parse_text_record(
|
|
||||||
self, file_name: str, file_timestamp: datetime.datetime, content: bytes
|
|
||||||
) -> dict:
|
|
||||||
if not content:
|
|
||||||
raise ValueError("empty plaintext tombstone")
|
|
||||||
tombstone_dict = {
|
tombstone_dict = {
|
||||||
"file_name": file_name,
|
"file_name": file_name,
|
||||||
"file_timestamp": convert_datetime_to_iso(file_timestamp),
|
"file_timestamp": convert_datetime_to_iso(file_timestamp),
|
||||||
}
|
}
|
||||||
lines = content.decode("utf-8", errors="replace").splitlines()
|
lines = content.decode("utf-8").splitlines()
|
||||||
for line_num, line in enumerate(lines, 1):
|
for line_num, line in enumerate(lines, 1):
|
||||||
if not line.strip() or TOMBSTONE_DELIMITER in line:
|
if not line.strip() or TOMBSTONE_DELIMITER in line:
|
||||||
continue
|
continue
|
||||||
@@ -189,7 +171,7 @@ class TombstoneCrashArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
# Validate the tombstone and add it to the results
|
# Validate the tombstone and add it to the results
|
||||||
tombstone = TombstoneCrashResult.model_validate(tombstone_dict)
|
tombstone = TombstoneCrashResult.model_validate(tombstone_dict)
|
||||||
return tombstone.model_dump()
|
self.results.append(tombstone.model_dump())
|
||||||
|
|
||||||
def _parse_tombstone_line(
|
def _parse_tombstone_line(
|
||||||
self, line: str, key: str, destination_key: str, tombstone: dict
|
self, line: str, key: str, destination_key: str, tombstone: dict
|
||||||
@@ -209,18 +191,11 @@ class TombstoneCrashArtifact(AndroidArtifact):
|
|||||||
def _load_key_value_line(
|
def _load_key_value_line(
|
||||||
self, line: str, key: str, destination_key: str, tombstone: dict
|
self, line: str, key: str, destination_key: str, tombstone: dict
|
||||||
) -> bool:
|
) -> bool:
|
||||||
# The caller matched the key as a bare prefix, so a longer word starting
|
line_key, value = line.split(":", 1)
|
||||||
# with it arrives here: `Caused by: …` inside an abort message reaches
|
if line_key != key:
|
||||||
# the `Cause` key. That is a different line, not a broken file — say so
|
raise ValueError(f"Expected key {key}, got {line_key}")
|
||||||
# by declining it, and let the remaining keys have their turn. Raising
|
|
||||||
# here discarded the whole tombstone, crash and stack trace included.
|
|
||||||
line_key, separator, value = line.partition(":")
|
|
||||||
if not separator or line_key != key:
|
|
||||||
return False
|
|
||||||
|
|
||||||
value_clean = value.strip()
|
value_clean = value.strip().strip("'")
|
||||||
if len(value_clean) >= 2 and value_clean[0] == value_clean[-1] == "'":
|
|
||||||
value_clean = value_clean[1:-1]
|
|
||||||
if destination_key == "uid":
|
if destination_key == "uid":
|
||||||
tombstone[destination_key] = int(value_clean)
|
tombstone[destination_key] = int(value_clean)
|
||||||
elif destination_key == "process_uptime":
|
elif destination_key == "process_uptime":
|
||||||
@@ -294,7 +269,9 @@ class TombstoneCrashArtifact(AndroidArtifact):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_timestamp_string(timestamp: str) -> str:
|
def _parse_timestamp_string(timestamp: str) -> str:
|
||||||
timestamp_parsed = parser.parse(timestamp)
|
timestamp_parsed = parser.parse(timestamp)
|
||||||
return convert_datetime_to_iso(timestamp_parsed)
|
# Preserve the source wall-clock time while returning the project-wide ISO format.
|
||||||
|
local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc)
|
||||||
|
return convert_datetime_to_iso(local_timestamp)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _proccess_name_from_thread(tombstone_dict: dict) -> str:
|
def _proccess_name_from_thread(tombstone_dict: dict) -> str:
|
||||||
|
|||||||
+72
-49
@@ -14,6 +14,13 @@ from mvt.common.cli_plugins import (
|
|||||||
load_cli_commands_option,
|
load_cli_commands_option,
|
||||||
register_cli_plugins,
|
register_cli_plugins,
|
||||||
)
|
)
|
||||||
|
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||||
|
from mvt.common.completion import (
|
||||||
|
SUPPORTED_SHELLS,
|
||||||
|
completion_instructions,
|
||||||
|
generate_completion_script,
|
||||||
|
install_completion_script,
|
||||||
|
)
|
||||||
from mvt.common.help import (
|
from mvt.common.help import (
|
||||||
HELP_MSG_ANDROID_BACKUP_PASSWORD,
|
HELP_MSG_ANDROID_BACKUP_PASSWORD,
|
||||||
HELP_MSG_CHECK_ADB_REMOVED,
|
HELP_MSG_CHECK_ADB_REMOVED,
|
||||||
@@ -24,6 +31,7 @@ from mvt.common.help import (
|
|||||||
HELP_MSG_CHECK_IOCS,
|
HELP_MSG_CHECK_IOCS,
|
||||||
HELP_MSG_CHECK_INTRUSION_LOGS,
|
HELP_MSG_CHECK_INTRUSION_LOGS,
|
||||||
HELP_MSG_DELAY_CHECKS,
|
HELP_MSG_DELAY_CHECKS,
|
||||||
|
HELP_MSG_COMPLETION,
|
||||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
HELP_MSG_DISABLE_UPDATE_CHECK,
|
||||||
HELP_MSG_HASHES,
|
HELP_MSG_HASHES,
|
||||||
@@ -35,16 +43,23 @@ from mvt.common.help import (
|
|||||||
HELP_MSG_OUTPUT,
|
HELP_MSG_OUTPUT,
|
||||||
HELP_MSG_STIX2,
|
HELP_MSG_STIX2,
|
||||||
HELP_MSG_VERBOSE,
|
HELP_MSG_VERBOSE,
|
||||||
HELP_MSG_VERBOSE_COMMAND,
|
|
||||||
HELP_MSG_VERSION,
|
HELP_MSG_VERSION,
|
||||||
HELP_MSG_VIRUS_TOTAL,
|
HELP_MSG_VIRUS_TOTAL,
|
||||||
)
|
)
|
||||||
|
from mvt.common.logo import logo
|
||||||
|
from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
|
||||||
|
from mvt.common.updates import IndicatorsUpdates
|
||||||
from mvt.common.utils import init_logging, set_verbose_logging
|
from mvt.common.utils import init_logging, set_verbose_logging
|
||||||
|
|
||||||
# The commands import what they run only when they are invoked. This module is
|
from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
|
||||||
# imported at every start of mvt-android, including by shell completion on
|
from .cmd_check_backup import CmdAndroidCheckBackup
|
||||||
# every keystroke, so importing it must do no more than build the command tree:
|
from .cmd_check_bugreport import CmdAndroidCheckBugreport
|
||||||
# the forensic modules, the backup parsers and the update checks stay out of it.
|
from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
|
||||||
|
from .modules.intrusion_logs import INTRUSION_LOGS_MODULES
|
||||||
|
from .modules.androidqf import ANDROIDQF_MODULES
|
||||||
|
from .modules.backup import BACKUP_MODULES
|
||||||
|
from .modules.backup.helpers import cli_load_android_backup_password
|
||||||
|
from .modules.bugreport import BUGREPORT_MODULES
|
||||||
|
|
||||||
init_logging()
|
init_logging()
|
||||||
log = logging.getLogger("mvt")
|
log = logging.getLogger("mvt")
|
||||||
@@ -62,14 +77,7 @@ def _get_disable_flags(ctx):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_verbose(ctx):
|
|
||||||
"""Return whether --verbose was passed to the CLI itself."""
|
|
||||||
return bool(ctx.obj and ctx.obj.get("verbose", False))
|
|
||||||
|
|
||||||
|
|
||||||
def _load_custom_modules(load_module):
|
def _load_custom_modules(load_module):
|
||||||
from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return load_custom_modules(load_module)
|
return load_custom_modules(load_module)
|
||||||
except CustomModuleLoadError as exc:
|
except CustomModuleLoadError as exc:
|
||||||
@@ -89,31 +97,60 @@ def _load_custom_modules(load_module):
|
|||||||
is_flag=True,
|
is_flag=True,
|
||||||
help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||||
)
|
)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def cli(ctx, disable_update_check, disable_indicator_update_check, verbose):
|
def cli(ctx, disable_update_check, disable_indicator_update_check):
|
||||||
ctx.ensure_object(dict)
|
ctx.ensure_object(dict)
|
||||||
ctx.obj["disable_version_check"] = disable_update_check
|
ctx.obj["disable_version_check"] = disable_update_check
|
||||||
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
||||||
ctx.obj["verbose"] = verbose
|
if ctx.invoked_subcommand != "completion":
|
||||||
set_verbose_logging(verbose)
|
logo(
|
||||||
|
disable_version_check=disable_update_check,
|
||||||
from mvt.common.logo import logo
|
disable_indicator_check=disable_indicator_update_check,
|
||||||
|
)
|
||||||
logo(
|
|
||||||
disable_version_check=disable_update_check,
|
|
||||||
disable_indicator_check=disable_indicator_update_check,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# Command: version
|
# Command: version
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION)
|
@cli.command("version", help=HELP_MSG_VERSION)
|
||||||
def version():
|
def version():
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Command: completion
|
||||||
|
# ==============================================================================
|
||||||
|
@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION)
|
||||||
|
@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS))
|
||||||
|
@click.option(
|
||||||
|
"--install",
|
||||||
|
is_flag=True,
|
||||||
|
help="Write completion files and update shell configuration.",
|
||||||
|
)
|
||||||
|
@click.pass_context
|
||||||
|
def completion(ctx, shell, install):
|
||||||
|
program_name = "mvt-android"
|
||||||
|
|
||||||
|
if shell is None:
|
||||||
|
if install:
|
||||||
|
raise click.UsageError("A shell is required when using --install.")
|
||||||
|
click.echo(completion_instructions(program_name))
|
||||||
|
return
|
||||||
|
|
||||||
|
root_cli = ctx.find_root().command
|
||||||
|
|
||||||
|
if install:
|
||||||
|
script_path = install_completion_script(root_cli, program_name, shell)
|
||||||
|
click.echo(f"Installed {shell} completion to {script_path}")
|
||||||
|
if shell in ("bash", "zsh"):
|
||||||
|
click.echo(f"Updated ~/.{shell}rc")
|
||||||
|
else:
|
||||||
|
click.echo("Fish loads completion files automatically.")
|
||||||
|
return
|
||||||
|
|
||||||
|
click.echo(generate_completion_script(root_cli, program_name, shell))
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# Command: check-adb (removed)
|
# Command: check-adb (removed)
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@@ -150,7 +187,7 @@ def check_adb(ctx):
|
|||||||
default=[],
|
default=[],
|
||||||
help=HELP_MSG_LOAD_MODULE,
|
help=HELP_MSG_LOAD_MODULE,
|
||||||
)
|
)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("BUGREPORT_PATH", type=click.Path(exists=True))
|
@click.argument("BUGREPORT_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_bugreport(
|
def check_bugreport(
|
||||||
@@ -163,9 +200,7 @@ def check_bugreport(
|
|||||||
verbose,
|
verbose,
|
||||||
bugreport_path,
|
bugreport_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_bugreport import CmdAndroidCheckBugreport
|
set_verbose_logging(verbose)
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
# Always generate hashes as bug reports are small.
|
# Always generate hashes as bug reports are small.
|
||||||
cmd = CmdAndroidCheckBugreport(
|
cmd = CmdAndroidCheckBugreport(
|
||||||
@@ -220,7 +255,7 @@ def check_bugreport(
|
|||||||
)
|
)
|
||||||
@click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
|
@click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
|
||||||
@click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
|
@click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_backup(
|
def check_backup(
|
||||||
@@ -234,10 +269,7 @@ def check_backup(
|
|||||||
verbose,
|
verbose,
|
||||||
backup_path,
|
backup_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_backup import CmdAndroidCheckBackup
|
set_verbose_logging(verbose)
|
||||||
from .modules.backup.helpers import cli_load_android_backup_password
|
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
|
|
||||||
# Always generate hashes as backups are generally small.
|
# Always generate hashes as backups are generally small.
|
||||||
@@ -297,7 +329,7 @@ def check_backup(
|
|||||||
)
|
)
|
||||||
@click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
|
@click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
|
||||||
@click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
|
@click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("ANDROIDQF_PATH", type=click.Path(exists=True))
|
@click.argument("ANDROIDQF_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_androidqf(
|
def check_androidqf(
|
||||||
@@ -315,10 +347,7 @@ def check_androidqf(
|
|||||||
verbose,
|
verbose,
|
||||||
androidqf_path,
|
androidqf_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
|
set_verbose_logging(verbose)
|
||||||
from .modules.backup.helpers import cli_load_android_backup_password
|
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
|
|
||||||
cmd = CmdAndroidCheckAndroidQF(
|
cmd = CmdAndroidCheckAndroidQF(
|
||||||
@@ -386,7 +415,7 @@ def check_androidqf(
|
|||||||
"time instead of UTC."
|
"time instead of UTC."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("LOGS_PATH", type=click.Path(exists=True))
|
@click.argument("LOGS_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_intrusion_logs(
|
def check_intrusion_logs(
|
||||||
@@ -400,9 +429,7 @@ def check_intrusion_logs(
|
|||||||
verbose,
|
verbose,
|
||||||
logs_path,
|
logs_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_intrusion_logs import CmdAndroidCheckIntrusionLogs
|
set_verbose_logging(verbose)
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
|
|
||||||
module_options = {}
|
module_options = {}
|
||||||
@@ -455,10 +482,6 @@ def check_intrusion_logs(
|
|||||||
@click.argument("FOLDER", type=click.Path(exists=True))
|
@click.argument("FOLDER", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
||||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
|
||||||
|
|
||||||
from .command_modules import ANDROID_CHECK_IOCS_MODULES
|
|
||||||
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
cmd = CmdCheckIOCS(
|
cmd = CmdCheckIOCS(
|
||||||
target_path=folder,
|
target_path=folder,
|
||||||
@@ -469,7 +492,9 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
|||||||
custom_modules=custom_modules,
|
custom_modules=custom_modules,
|
||||||
platform="android",
|
platform="android",
|
||||||
)
|
)
|
||||||
cmd.modules = ANDROID_CHECK_IOCS_MODULES
|
cmd.modules = (
|
||||||
|
BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES
|
||||||
|
)
|
||||||
|
|
||||||
if list_modules:
|
if list_modules:
|
||||||
cmd.list_modules()
|
cmd.list_modules()
|
||||||
@@ -485,8 +510,6 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
|||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
|
@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
|
||||||
def download_indicators():
|
def download_indicators():
|
||||||
from mvt.common.updates import IndicatorsUpdates
|
|
||||||
|
|
||||||
ioc_updates = IndicatorsUpdates()
|
ioc_updates = IndicatorsUpdates()
|
||||||
ioc_updates.update()
|
ioc_updates.update()
|
||||||
|
|
||||||
|
|||||||
@@ -292,7 +292,9 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
try:
|
try:
|
||||||
cmd.from_ab(backup)
|
cmd.from_ab(backup)
|
||||||
except InvalidAndroidBackup as exc:
|
except InvalidAndroidBackup as exc:
|
||||||
self.log.warning("Skipping backup modules: %s", exc)
|
self.log.warning(
|
||||||
|
"Skipping backup modules as backup.ab is malformed: %s", exc
|
||||||
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
cmd.run()
|
cmd.run()
|
||||||
|
|||||||
@@ -87,15 +87,11 @@ class CmdAndroidCheckBackup(Command):
|
|||||||
if header["encryption"] != "none":
|
if header["encryption"] != "none":
|
||||||
password = prompt_or_load_android_backup_password(log, self.module_options)
|
password = prompt_or_load_android_backup_password(log, self.module_options)
|
||||||
if not password:
|
if not password:
|
||||||
if self.sub_command:
|
|
||||||
raise InvalidAndroidBackup("No backup password provided")
|
|
||||||
log.critical("No backup password provided.")
|
log.critical("No backup password provided.")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
try:
|
try:
|
||||||
tardata = parse_backup_file(ab_file_bytes, password=password)
|
tardata = parse_backup_file(ab_file_bytes, password=password)
|
||||||
except InvalidBackupPassword:
|
except InvalidBackupPassword:
|
||||||
if self.sub_command:
|
|
||||||
raise InvalidAndroidBackup("Invalid backup password")
|
|
||||||
log.critical("Invalid backup password")
|
log.critical("Invalid backup password")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
except AndroidBackupParsingError as exc:
|
except AndroidBackupParsingError as exc:
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
"""Module lists an mvt-android command composes from more than one family.
|
|
||||||
|
|
||||||
Commands whose modules are one family read that family directly. check-iocs
|
|
||||||
re-checks stored results, so it has to know every module that could have
|
|
||||||
written one, and both the CLI and any other code needing that answer share
|
|
||||||
the list from here rather than each concatenating their own.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .modules.androidqf import ANDROIDQF_MODULES
|
|
||||||
from .modules.backup import BACKUP_MODULES
|
|
||||||
from .modules.bugreport import BUGREPORT_MODULES
|
|
||||||
from .modules.intrusion_logs import INTRUSION_LOGS_MODULES
|
|
||||||
|
|
||||||
ANDROID_CHECK_IOCS_MODULES: list[type[MVTModule]] = (
|
|
||||||
BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES + INTRUSION_LOGS_MODULES
|
|
||||||
)
|
|
||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .aqf_files import AQFFiles
|
from .aqf_files import AQFFiles
|
||||||
from .aqf_getprop import AQFGetProp
|
from .aqf_getprop import AQFGetProp
|
||||||
from .aqf_log_timestamps import AQFLogTimestamps
|
from .aqf_log_timestamps import AQFLogTimestamps
|
||||||
@@ -14,7 +12,7 @@ from .aqf_settings import AQFSettings
|
|||||||
from .mounts import Mounts
|
from .mounts import Mounts
|
||||||
from .root_binaries import RootBinaries
|
from .root_binaries import RootBinaries
|
||||||
|
|
||||||
ANDROIDQF_MODULES: list[type[MVTModule]] = [
|
ANDROIDQF_MODULES = [
|
||||||
AQFPackages,
|
AQFPackages,
|
||||||
AQFProcesses,
|
AQFProcesses,
|
||||||
AQFGetProp,
|
AQFGetProp,
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .sms import SMS
|
from .sms import SMS
|
||||||
|
|
||||||
BACKUP_MODULES: list[type[MVTModule]] = [SMS]
|
BACKUP_MODULES = [SMS]
|
||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .dumpsys_accessibility import DumpsysAccessibility
|
from .dumpsys_accessibility import DumpsysAccessibility
|
||||||
from .dumpsys_activities import DumpsysActivities
|
from .dumpsys_activities import DumpsysActivities
|
||||||
from .dumpsys_appops import DumpsysAppops
|
from .dumpsys_appops import DumpsysAppops
|
||||||
@@ -18,11 +16,8 @@ from .dumpsys_receivers import DumpsysReceivers
|
|||||||
from .dumpsys_adb_state import DumpsysADBState
|
from .dumpsys_adb_state import DumpsysADBState
|
||||||
from .fs_timestamps import BugReportTimestamps
|
from .fs_timestamps import BugReportTimestamps
|
||||||
from .tombstones import Tombstones
|
from .tombstones import Tombstones
|
||||||
from .mounts import Mounts
|
|
||||||
from .processes import Processes
|
|
||||||
from .settings import Settings
|
|
||||||
|
|
||||||
BUGREPORT_MODULES: list[type[MVTModule]] = [
|
BUGREPORT_MODULES = [
|
||||||
DumpsysAccessibility,
|
DumpsysAccessibility,
|
||||||
DumpsysActivities,
|
DumpsysActivities,
|
||||||
DumpsysAppops,
|
DumpsysAppops,
|
||||||
@@ -36,7 +31,4 @@ BUGREPORT_MODULES: list[type[MVTModule]] = [
|
|||||||
DumpsysADBState,
|
DumpsysADBState,
|
||||||
BugReportTimestamps,
|
BugReportTimestamps,
|
||||||
Tombstones,
|
Tombstones,
|
||||||
Processes,
|
|
||||||
Settings,
|
|
||||||
Mounts,
|
|
||||||
]
|
]
|
||||||
@@ -72,11 +72,7 @@ class BugReportModule(MVTModule):
|
|||||||
if not self.extract_path:
|
if not self.extract_path:
|
||||||
raise ValueError("extract_path is not set")
|
raise ValueError("extract_path is not set")
|
||||||
joined = os.path.join(self.extract_path, file_path)
|
joined = os.path.join(self.extract_path, file_path)
|
||||||
if (
|
if not Path(joined).resolve().is_relative_to(Path(self.extract_path).resolve()):
|
||||||
not Path(joined)
|
|
||||||
.resolve()
|
|
||||||
.is_relative_to(Path(self.extract_path).resolve())
|
|
||||||
):
|
|
||||||
raise ValueError("unsafe file_path")
|
raise ValueError("unsafe file_path")
|
||||||
handle = open(joined, "rb")
|
handle = open(joined, "rb")
|
||||||
|
|
||||||
@@ -104,27 +100,6 @@ class BugReportModule(MVTModule):
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def extract_command_section(content: str, heading: str) -> str:
|
|
||||||
"""Return a bugreport command section without consuming the next one.
|
|
||||||
|
|
||||||
Bugreport separators include timing text, so looking for a line equal to
|
|
||||||
``------`` is not sufficient and can accidentally feed the remainder of
|
|
||||||
dumpstate to a parser.
|
|
||||||
"""
|
|
||||||
lines: list[str] = []
|
|
||||||
in_section = False
|
|
||||||
for line in content.splitlines():
|
|
||||||
stripped = line.strip()
|
|
||||||
if not in_section:
|
|
||||||
if stripped.startswith(heading):
|
|
||||||
in_section = True
|
|
||||||
continue
|
|
||||||
if stripped.startswith("------"):
|
|
||||||
break
|
|
||||||
lines.append(line)
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
def _get_file_modification_time(self, file_path: str) -> datetime.datetime:
|
def _get_file_modification_time(self, file_path: str) -> datetime.datetime:
|
||||||
if self.zip_archive:
|
if self.zip_archive:
|
||||||
file_timetuple = self.zip_archive.getinfo(file_path).date_time
|
file_timetuple = self.zip_archive.getinfo(file_path).date_time
|
||||||
|
|||||||
@@ -49,7 +49,9 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule):
|
|||||||
self.parse(content)
|
self.parse(content)
|
||||||
|
|
||||||
for result in self.results:
|
for result in self.results:
|
||||||
self.log.info('Found accessibility service "%s"', result.get("component"))
|
self.log.info(
|
||||||
|
'Found installed accessibility service "%s"', result.get("service")
|
||||||
|
)
|
||||||
|
|
||||||
self.log.info(
|
self.log.info(
|
||||||
"Identified a total of %d accessibility services", len(self.results)
|
"Identified a total of %d accessibility services", len(self.results)
|
||||||
|
|||||||
@@ -44,8 +44,21 @@ class DumpsysGetProp(GetPropArtifact, BugReportModule):
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
section = self.extract_command_section(
|
lines = []
|
||||||
content.decode(errors="ignore"), "------ SYSTEM PROPERTIES"
|
in_getprop = False
|
||||||
)
|
|
||||||
self.parse(section)
|
for line in content.decode(errors="ignore").splitlines():
|
||||||
|
if line.strip().startswith("------ SYSTEM PROPERTIES"):
|
||||||
|
in_getprop = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not in_getprop:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if line.strip() == "------":
|
||||||
|
break
|
||||||
|
|
||||||
|
lines.append(line)
|
||||||
|
|
||||||
|
self.parse("\n".join(lines))
|
||||||
self.log.info("Extracted %d Android system properties", len(self.results))
|
self.log.info("Extracted %d Android system properties", len(self.results))
|
||||||
@@ -50,10 +50,7 @@ class DumpsysPackages(DumpsysPackagesArtifact, BugReportModule):
|
|||||||
|
|
||||||
for result in self.results:
|
for result in self.results:
|
||||||
dangerous_permissions_count = 0
|
dangerous_permissions_count = 0
|
||||||
permissions = list(result["permissions"])
|
for perm in result["permissions"]:
|
||||||
for user in result.get("users", []):
|
|
||||||
permissions.extend(user.get("permissions", []))
|
|
||||||
for perm in permissions:
|
|
||||||
if perm["name"] in DANGEROUS_PERMISSIONS:
|
if perm["name"] in DANGEROUS_PERMISSIONS:
|
||||||
dangerous_permissions_count += 1
|
dangerous_permissions_count += 1
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from mvt.common.module_types import ModuleResults
|
|||||||
|
|
||||||
|
|
||||||
class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule):
|
class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule):
|
||||||
"""This module extracts raw per-package compatibility overrides."""
|
"""This module extracts details on uninstalled apps."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -48,4 +48,4 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule):
|
|||||||
)
|
)
|
||||||
self.parse(content)
|
self.parse(content)
|
||||||
|
|
||||||
self.log.info("Found %d package compatibility overrides", len(self.results))
|
self.log.info("Found %d uninstalled apps", len(self.results))
|
||||||
@@ -33,7 +33,7 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule):
|
|||||||
results=results,
|
results=results,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.results = results if results else []
|
self.results = results if results else {}
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
content = self._get_dumpstate_file()
|
content = self._get_dumpstate_file()
|
||||||
@@ -49,4 +49,4 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.parse(dumpsys_section)
|
self.parse(dumpsys_section)
|
||||||
self.log.info("Extracted %d package receivers", len(self.results))
|
self.log.info("Extracted receivers for %d intents", len(self.results))
|
||||||
@@ -4,15 +4,12 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import datetime
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
||||||
|
|
||||||
from mvt.common.utils import convert_datetime_to_iso
|
from mvt.common.utils import convert_datetime_to_iso
|
||||||
from .base import BugReportModule
|
from .base import BugReportModule
|
||||||
from mvt.common.module_types import ModuleResults
|
from mvt.common.module_types import ModuleResults
|
||||||
from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
|
from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
|
||||||
from mvt.android.artifacts.getprop import GetProp
|
|
||||||
|
|
||||||
|
|
||||||
class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
|
class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
|
||||||
@@ -41,44 +38,15 @@ class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
|
|||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
filesystem_files = self._get_files_by_pattern("FS/*")
|
filesystem_files = self._get_files_by_pattern("FS/*")
|
||||||
|
|
||||||
timezone_name = None
|
|
||||||
dumpstate = self._get_dumpstate_file()
|
|
||||||
if dumpstate:
|
|
||||||
section = self.extract_command_section(
|
|
||||||
dumpstate.decode("utf-8", errors="replace"),
|
|
||||||
"------ SYSTEM PROPERTIES",
|
|
||||||
)
|
|
||||||
properties = GetProp()
|
|
||||||
properties.parse(section)
|
|
||||||
timezone_name = properties.get_device_timezone()
|
|
||||||
timezone = None
|
|
||||||
if timezone_name:
|
|
||||||
try:
|
|
||||||
timezone = ZoneInfo(timezone_name)
|
|
||||||
except ZoneInfoNotFoundError:
|
|
||||||
self.log.warning("Unknown device timezone %s", timezone_name)
|
|
||||||
|
|
||||||
self.results = []
|
self.results = []
|
||||||
for file in filesystem_files:
|
for file in filesystem_files:
|
||||||
# Only the modification time is available in the zip file metadata.
|
# Only the modification time is available in the zip file metadata.
|
||||||
# The timezone is the local timezone of the machine the phone.
|
# The timezone is the local timezone of the machine the phone.
|
||||||
modification_time = self._get_file_modification_time(file)
|
modification_time = self._get_file_modification_time(file)
|
||||||
utc_time = None
|
|
||||||
if timezone is not None:
|
|
||||||
utc_time = convert_datetime_to_iso(
|
|
||||||
modification_time.replace(tzinfo=timezone).astimezone(
|
|
||||||
datetime.timezone.utc
|
|
||||||
)
|
|
||||||
)
|
|
||||||
self.results.append(
|
self.results.append(
|
||||||
{
|
{
|
||||||
"path": file,
|
"path": file,
|
||||||
"modified_time": convert_datetime_to_iso(modification_time),
|
"modified_time": convert_datetime_to_iso(modification_time),
|
||||||
"modified_time_utc": utc_time,
|
|
||||||
"timezone": timezone_name,
|
|
||||||
"timestamp_source": (
|
|
||||||
"zip_metadata" if self.zip_archive else "filesystem_metadata"
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,41 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from mvt.android.artifacts.mounts import Mounts as MountsArtifact
|
|
||||||
|
|
||||||
from .base import BugReportModule
|
|
||||||
|
|
||||||
|
|
||||||
class Mounts(MountsArtifact, BugReportModule):
|
|
||||||
"""Extract and deduplicate process mount namespaces from mountinfo."""
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
mount_files = self._get_files_by_pattern("FS/proc/*/mountinfo")
|
|
||||||
if not mount_files:
|
|
||||||
mount_files = self._get_files_by_pattern("*/proc/*/mountinfo")
|
|
||||||
unique: dict[tuple, dict] = {}
|
|
||||||
for file_path in mount_files:
|
|
||||||
pid_match = re.search(r"/proc/(\d+)/mountinfo$", file_path)
|
|
||||||
if not pid_match:
|
|
||||||
continue
|
|
||||||
entries = self.parse_mountinfo(
|
|
||||||
self._get_file_content(file_path).decode("utf-8", errors="replace"),
|
|
||||||
int(pid_match.group(1)),
|
|
||||||
)
|
|
||||||
for entry in entries:
|
|
||||||
identity = (
|
|
||||||
entry["major_minor"],
|
|
||||||
entry["root"],
|
|
||||||
entry["mount_point"],
|
|
||||||
entry["device"],
|
|
||||||
entry["filesystem_type"],
|
|
||||||
entry["mount_options"],
|
|
||||||
)
|
|
||||||
if identity in unique:
|
|
||||||
unique[identity]["process_ids"].extend(entry["process_ids"])
|
|
||||||
else:
|
|
||||||
unique[identity] = entry
|
|
||||||
self.results = list(unique.values())
|
|
||||||
self.log.info("Extracted %d unique mount records", len(self.results))
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
|
|
||||||
from mvt.android.artifacts.processes import Processes as ProcessesArtifact
|
|
||||||
|
|
||||||
from .base import BugReportModule
|
|
||||||
|
|
||||||
|
|
||||||
class Processes(ProcessesArtifact, BugReportModule):
|
|
||||||
"""Extract the process and thread table from dumpstate."""
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
data = self._get_dumpstate_file()
|
|
||||||
if not data:
|
|
||||||
self.log.error("Unable to find dumpstate file")
|
|
||||||
return
|
|
||||||
section = self.extract_command_section(
|
|
||||||
data.decode("utf-8", errors="replace"),
|
|
||||||
"------ PROCESSES AND THREADS",
|
|
||||||
)
|
|
||||||
self.parse(section)
|
|
||||||
self.log.info("Identified %d running process threads", len(self.results))
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
|
|
||||||
from mvt.android.artifacts.settings import Settings as SettingsArtifact
|
|
||||||
|
|
||||||
from .base import BugReportModule
|
|
||||||
|
|
||||||
|
|
||||||
class Settings(SettingsArtifact, BugReportModule):
|
|
||||||
"""Extract all SettingsProvider namespaces and users."""
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
data = self._get_dumpstate_file()
|
|
||||||
if not data:
|
|
||||||
self.log.error("Unable to find dumpstate file")
|
|
||||||
return
|
|
||||||
section = self.extract_dumpsys_section(
|
|
||||||
data.decode("utf-8", errors="replace"), "DUMP OF SERVICE settings:"
|
|
||||||
)
|
|
||||||
self.parse(section)
|
|
||||||
count = sum(len(settings) for settings in self.results.values())
|
|
||||||
self.log.info("Identified %d Android settings", count)
|
|
||||||
@@ -43,76 +43,21 @@ class Tombstones(TombstoneCrashArtifact, BugReportModule):
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
grouped: dict[str, dict[str, str]] = {}
|
for tombstone_file in sorted(tombstone_files):
|
||||||
for file_path in tombstone_files:
|
tombstone_filename = tombstone_file.split("/")[-1]
|
||||||
file_name = file_path.rsplit("/", 1)[-1]
|
modification_time = self._get_file_modification_time(tombstone_file)
|
||||||
source = "protobuf" if file_name.endswith(".pb") else "text"
|
tombstone_data = self._get_file_content(tombstone_file)
|
||||||
crash_id = file_name.removesuffix(".pb")
|
|
||||||
grouped.setdefault(crash_id, {})[source] = file_path
|
|
||||||
|
|
||||||
for crash_id, paths in sorted(grouped.items()):
|
try:
|
||||||
parsed_sources: dict[str, dict] = {}
|
if tombstone_file.endswith(".pb"):
|
||||||
source_records: dict[str, dict] = {}
|
self.parse_protobuf(
|
||||||
for source in ("text", "protobuf"):
|
tombstone_filename, modification_time, tombstone_data
|
||||||
file_path = paths.get(source)
|
|
||||||
if file_path is None:
|
|
||||||
continue
|
|
||||||
file_name = file_path.rsplit("/", 1)[-1]
|
|
||||||
file_timestamp = self._get_file_modification_time(file_path)
|
|
||||||
source_info = {
|
|
||||||
"file_name": file_name,
|
|
||||||
"file_timestamp": file_timestamp.isoformat(),
|
|
||||||
"parsed": False,
|
|
||||||
"error": None,
|
|
||||||
"record": None,
|
|
||||||
}
|
|
||||||
try:
|
|
||||||
data = self._get_file_content(file_path)
|
|
||||||
if source == "protobuf":
|
|
||||||
record = self.parse_protobuf_record(
|
|
||||||
file_name, file_timestamp, data
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
record = self.parse_text_record(file_name, file_timestamp, data)
|
|
||||||
source_info["parsed"] = True
|
|
||||||
source_info["record"] = record
|
|
||||||
source_records[source] = record
|
|
||||||
except Exception as exc:
|
|
||||||
source_info["error"] = str(exc)
|
|
||||||
self.log.error(
|
|
||||||
"Error parsing tombstone file %s: %s", file_path, exc
|
|
||||||
)
|
)
|
||||||
parsed_sources[source] = source_info
|
else:
|
||||||
|
self.parse(tombstone_filename, modification_time, tombstone_data)
|
||||||
if not source_records:
|
except ValueError as e:
|
||||||
continue
|
# Catch any exceptions raised during parsing or validation.
|
||||||
preferred = source_records.get("protobuf") or source_records["text"]
|
self.log.error(f"Error parsing tombstone file {tombstone_file}: {e}")
|
||||||
canonical = dict(preferred)
|
|
||||||
text_record = source_records.get("text")
|
|
||||||
if text_record:
|
|
||||||
for key, value in text_record.items():
|
|
||||||
if canonical.get(key) in (None, "", [], {}):
|
|
||||||
canonical[key] = value
|
|
||||||
|
|
||||||
differences = {}
|
|
||||||
protobuf_record = source_records.get("protobuf")
|
|
||||||
if text_record and protobuf_record:
|
|
||||||
for key in text_record.keys() & protobuf_record.keys():
|
|
||||||
if key in ("file_name", "file_timestamp"):
|
|
||||||
continue
|
|
||||||
if text_record[key] != protobuf_record[key]:
|
|
||||||
differences[key] = {
|
|
||||||
"text": text_record[key],
|
|
||||||
"protobuf": protobuf_record[key],
|
|
||||||
}
|
|
||||||
canonical.update(
|
|
||||||
{
|
|
||||||
"crash_id": crash_id,
|
|
||||||
"sources": parsed_sources,
|
|
||||||
"differences": differences,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
self.results.append(canonical)
|
|
||||||
|
|
||||||
self.log.info(
|
self.log.info(
|
||||||
"Extracted a total of %d tombstone files",
|
"Extracted a total of %d tombstone files",
|
||||||
|
|||||||
@@ -3,13 +3,11 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .connect_event import ConnectEvent
|
from .connect_event import ConnectEvent
|
||||||
from .dns_event import DnsEvent
|
from .dns_event import DnsEvent
|
||||||
from .security_event import SecurityEvent
|
from .security_event import SecurityEvent
|
||||||
|
|
||||||
INTRUSION_LOGS_MODULES: list[type[MVTModule]] = [
|
INTRUSION_LOGS_MODULES = [
|
||||||
DnsEvent,
|
DnsEvent,
|
||||||
ConnectEvent,
|
ConnectEvent,
|
||||||
SecurityEvent,
|
SecurityEvent,
|
||||||
|
|||||||
-110
@@ -1,110 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
import click
|
|
||||||
|
|
||||||
from mvt.common.cli_plugins import (
|
|
||||||
MVT_CUSTOM_COMMANDS_ENV,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
load_cli_commands_option,
|
|
||||||
register_cli_plugins,
|
|
||||||
)
|
|
||||||
from mvt.common.cmd_plugins import plugins
|
|
||||||
from mvt.common.completion import completion
|
|
||||||
from mvt.common.help import (
|
|
||||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
|
||||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
|
||||||
HELP_MSG_STIX2,
|
|
||||||
HELP_MSG_VERBOSE,
|
|
||||||
HELP_MSG_VERSION,
|
|
||||||
)
|
|
||||||
from mvt.common.logo import logo
|
|
||||||
from mvt.common.updates import IndicatorsUpdates
|
|
||||||
from mvt.common.utils import init_logging, set_verbose_logging
|
|
||||||
|
|
||||||
init_logging()
|
|
||||||
|
|
||||||
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
|
||||||
# Main
|
|
||||||
# ==============================================================================
|
|
||||||
@click.group(invoke_without_command=True)
|
|
||||||
@load_cli_commands_option
|
|
||||||
@click.option(
|
|
||||||
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
|
||||||
)
|
|
||||||
@click.option(
|
|
||||||
"--disable-indicator-update-check",
|
|
||||||
is_flag=True,
|
|
||||||
help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
|
||||||
)
|
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
|
||||||
@click.pass_context
|
|
||||||
def cli(ctx, disable_update_check, disable_indicator_update_check, verbose):
|
|
||||||
"""Mobile Verification Toolkit.
|
|
||||||
|
|
||||||
The 'mvt-ios' and 'mvt-android' CLI commands are used to perform
|
|
||||||
forensic analysis on IOS and Android devices.
|
|
||||||
"""
|
|
||||||
ctx.ensure_object(dict)
|
|
||||||
ctx.obj["disable_version_check"] = disable_update_check
|
|
||||||
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
|
||||||
ctx.obj["verbose"] = verbose
|
|
||||||
set_verbose_logging(verbose)
|
|
||||||
if ctx.invoked_subcommand != "completion":
|
|
||||||
logo(
|
|
||||||
disable_version_check=disable_update_check,
|
|
||||||
disable_indicator_check=disable_indicator_update_check,
|
|
||||||
)
|
|
||||||
if ctx.invoked_subcommand is None:
|
|
||||||
click.echo(ctx.get_help())
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
|
||||||
# Command: download-iocs
|
|
||||||
# ==============================================================================
|
|
||||||
@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
|
|
||||||
def download_iocs():
|
|
||||||
ioc_updates = IndicatorsUpdates()
|
|
||||||
ioc_updates.update()
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
|
||||||
# Command: completion
|
|
||||||
# ==============================================================================
|
|
||||||
cli.add_command(completion)
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
|
||||||
# Command: version
|
|
||||||
# ==============================================================================
|
|
||||||
@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION)
|
|
||||||
def version():
|
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
# The plugins command is registered as a built-in command, before any external
|
|
||||||
# command, so that an installed package can never replace it.
|
|
||||||
cli.add_command(plugins)
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
|
||||||
# Entry point of the mvt console script
|
|
||||||
# ==============================================================================
|
|
||||||
def main() -> None:
|
|
||||||
"""Register the external commands and run the mvt CLI.
|
|
||||||
|
|
||||||
External commands are registered here rather than when this module is
|
|
||||||
imported, so that importing MVT never runs third-party code and a plugin
|
|
||||||
importing from MVT cannot re-enter a module that is still initializing.
|
|
||||||
"""
|
|
||||||
register_cli_plugins(
|
|
||||||
cli,
|
|
||||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
|
||||||
)
|
|
||||||
cli()
|
|
||||||
@@ -15,20 +15,10 @@ from typing import Iterable
|
|||||||
|
|
||||||
import click
|
import click
|
||||||
|
|
||||||
# This module is imported by every CLI at start-up, and by shell completion on
|
|
||||||
# every keystroke, so it must stay cheap: nothing here may import the module
|
|
||||||
# machinery (mvt.common.module_loader and what it pulls in).
|
|
||||||
|
|
||||||
IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins"
|
IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins"
|
||||||
ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins"
|
ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins"
|
||||||
# Commands in this group are registered on the platform-neutral mvt command only.
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP = "mvt.cli_plugins"
|
|
||||||
MVT_CUSTOM_COMMANDS_ENV = "MVT_CUSTOM_COMMANDS"
|
|
||||||
MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS"
|
MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS"
|
||||||
MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS"
|
MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS"
|
||||||
# Prefix of the import name given to a command file loaded from a path. Shared
|
|
||||||
# with module_loader, which recognises such files when naming their loggers.
|
|
||||||
CUSTOM_COMMAND_MODULE_PREFIX = "_mvt_custom_command_"
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -64,7 +54,7 @@ class BrokenPluginCommand(click.Command):
|
|||||||
|
|
||||||
def _module_name_for_path(path: Path) -> str:
|
def _module_name_for_path(path: Path) -> str:
|
||||||
digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16]
|
digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16]
|
||||||
return f"{CUSTOM_COMMAND_MODULE_PREFIX}{path.stem}_{digest}"
|
return f"_mvt_custom_command_{path.stem}_{digest}"
|
||||||
|
|
||||||
|
|
||||||
def _iter_command_files(path: Path) -> Iterable[Path]:
|
def _iter_command_files(path: Path) -> Iterable[Path]:
|
||||||
@@ -265,16 +255,6 @@ def register_cli_plugins(
|
|||||||
entry_point_group: str,
|
entry_point_group: str,
|
||||||
environment_variable: str,
|
environment_variable: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Register the external commands of one CLI on its group.
|
|
||||||
|
|
||||||
Each CLI has one entry-point group and one environment variable of its
|
|
||||||
own, so a command package chooses the CLIs its commands are added to.
|
|
||||||
|
|
||||||
:param group: CLI group to register the external commands on.
|
|
||||||
:param entry_point_group: Entry-point group of the CLI.
|
|
||||||
:param environment_variable: Name of the environment variable holding a
|
|
||||||
path to load commands from.
|
|
||||||
"""
|
|
||||||
environment_path = os.environ.get(environment_variable)
|
environment_path = os.environ.get(environment_variable)
|
||||||
if environment_path:
|
if environment_path:
|
||||||
register_cli_commands_from_path(group, environment_path)
|
register_cli_commands_from_path(group, environment_path)
|
||||||
|
|||||||
@@ -1,205 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
import importlib.metadata
|
|
||||||
import logging
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import click
|
|
||||||
from rich.console import Console
|
|
||||||
from rich.table import Table
|
|
||||||
|
|
||||||
from .cli_plugins import (
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
)
|
|
||||||
from .config import settings
|
|
||||||
from .help import (
|
|
||||||
HELP_MSG_PLUGINS,
|
|
||||||
HELP_MSG_PLUGINS_CHECK_UPDATES,
|
|
||||||
HELP_MSG_PLUGINS_LIST,
|
|
||||||
)
|
|
||||||
from .module import MVTModule
|
|
||||||
from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url
|
|
||||||
from .updates import (
|
|
||||||
SHORT_COMMIT_LENGTH,
|
|
||||||
PluginUpdates,
|
|
||||||
installed_plugin_distributions,
|
|
||||||
)
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
CLI_PLUGIN_GROUPS = (
|
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
)
|
|
||||||
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
|
||||||
|
|
||||||
|
|
||||||
def _entry_points(group: str) -> list[importlib.metadata.EntryPoint]:
|
|
||||||
try:
|
|
||||||
return list(importlib.metadata.entry_points(group=group))
|
|
||||||
except Exception as exc:
|
|
||||||
log.warning("Unable to discover the entry points in group %s: %s", group, exc)
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def _entry_point_distribution(
|
|
||||||
entry_point: importlib.metadata.EntryPoint,
|
|
||||||
) -> Optional[str]:
|
|
||||||
dist = getattr(entry_point, "dist", None)
|
|
||||||
if dist is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return dist.name
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _distribution_version(dist: importlib.metadata.Distribution) -> str:
|
|
||||||
try:
|
|
||||||
return dist.version or "unknown"
|
|
||||||
except Exception:
|
|
||||||
return "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
def _distribution_origin(dist: importlib.metadata.Distribution) -> str:
|
|
||||||
"""Describe where a plugin package was installed from."""
|
|
||||||
direct_url = distribution_direct_url(dist)
|
|
||||||
if direct_url is None:
|
|
||||||
return "pypi"
|
|
||||||
|
|
||||||
vcs_info = direct_url.get("vcs_info")
|
|
||||||
if isinstance(vcs_info, dict):
|
|
||||||
commit = vcs_info.get("commit_id") or ""
|
|
||||||
if commit:
|
|
||||||
return f"git+{commit[:SHORT_COMMIT_LENGTH]}"
|
|
||||||
return "git"
|
|
||||||
|
|
||||||
return "local"
|
|
||||||
|
|
||||||
|
|
||||||
def _contributed_modules(
|
|
||||||
entry_points: list[importlib.metadata.EntryPoint], distribution: str
|
|
||||||
) -> str:
|
|
||||||
"""Count the forensic modules a plugin package contributes.
|
|
||||||
|
|
||||||
Entry points are resolved the way MVT resolves them when it loads
|
|
||||||
modules, but a broken entry point is reported instead of raising: listing
|
|
||||||
the installed plugins must work even when one of them is faulty.
|
|
||||||
"""
|
|
||||||
count = 0
|
|
||||||
broken = False
|
|
||||||
|
|
||||||
for entry_point in entry_points:
|
|
||||||
if _entry_point_distribution(entry_point) != distribution:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
loaded = entry_point.load()
|
|
||||||
if callable(loaded) and not isinstance(loaded, type):
|
|
||||||
loaded = loaded()
|
|
||||||
count += sum(
|
|
||||||
1
|
|
||||||
for module in loaded
|
|
||||||
if isinstance(module, type) and issubclass(module, MVTModule)
|
|
||||||
)
|
|
||||||
except (Exception, SystemExit) as exc:
|
|
||||||
log.debug(
|
|
||||||
"Unable to load the modules of entry point %s (%s): %s",
|
|
||||||
entry_point.name,
|
|
||||||
entry_point.value,
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
broken = True
|
|
||||||
|
|
||||||
if broken:
|
|
||||||
return f"{count} (error)" if count else "error"
|
|
||||||
|
|
||||||
return str(count)
|
|
||||||
|
|
||||||
|
|
||||||
def _contributed_commands(
|
|
||||||
entry_points: list[importlib.metadata.EntryPoint], distribution: str
|
|
||||||
) -> str:
|
|
||||||
names = {
|
|
||||||
entry_point.name
|
|
||||||
for entry_point in entry_points
|
|
||||||
if _entry_point_distribution(entry_point) == distribution
|
|
||||||
}
|
|
||||||
|
|
||||||
return ", ".join(sorted(names)) if names else "-"
|
|
||||||
|
|
||||||
|
|
||||||
@click.group("plugins", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS)
|
|
||||||
def plugins() -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
@plugins.command("list", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_PLUGINS_LIST)
|
|
||||||
def list_plugins() -> None:
|
|
||||||
distributions = installed_plugin_distributions()
|
|
||||||
if not distributions:
|
|
||||||
click.echo("No MVT plugins are installed.")
|
|
||||||
return
|
|
||||||
|
|
||||||
module_entry_points = _entry_points(MODULES_ENTRY_POINT_GROUP)
|
|
||||||
command_entry_points = []
|
|
||||||
for group in CLI_PLUGIN_GROUPS:
|
|
||||||
command_entry_points.extend(_entry_points(group))
|
|
||||||
|
|
||||||
table = Table(title="Installed MVT plugins")
|
|
||||||
table.add_column("Name", style="bold")
|
|
||||||
table.add_column("Version")
|
|
||||||
table.add_column("Origin")
|
|
||||||
table.add_column("Modules", justify="right")
|
|
||||||
table.add_column("Commands")
|
|
||||||
|
|
||||||
for dist in distributions:
|
|
||||||
name = dist.name
|
|
||||||
table.add_row(
|
|
||||||
name,
|
|
||||||
_distribution_version(dist),
|
|
||||||
_distribution_origin(dist),
|
|
||||||
_contributed_modules(module_entry_points, name),
|
|
||||||
_contributed_commands(command_entry_points, name),
|
|
||||||
)
|
|
||||||
|
|
||||||
Console().print(table)
|
|
||||||
|
|
||||||
|
|
||||||
@plugins.command(
|
|
||||||
"check-updates",
|
|
||||||
context_settings=CONTEXT_SETTINGS,
|
|
||||||
help=HELP_MSG_PLUGINS_CHECK_UPDATES,
|
|
||||||
short_help="Check the installed plugins for updates",
|
|
||||||
)
|
|
||||||
def check_plugin_updates() -> None:
|
|
||||||
if not settings.NETWORK_ACCESS_ALLOWED:
|
|
||||||
click.echo(
|
|
||||||
"Network access is disabled, cannot check for plugin updates. "
|
|
||||||
"Enable NETWORK_ACCESS_ALLOWED in the MVT configuration to check."
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
if not installed_plugin_distributions():
|
|
||||||
click.echo("No MVT plugins are installed.")
|
|
||||||
return
|
|
||||||
|
|
||||||
findings = PluginUpdates().check()
|
|
||||||
if not findings:
|
|
||||||
click.echo("All plugins are up to date.")
|
|
||||||
return
|
|
||||||
|
|
||||||
click.echo("Plugin updates available:")
|
|
||||||
for finding in findings:
|
|
||||||
click.echo(f" {finding['name']} {finding['installed']} → {finding['latest']}")
|
|
||||||
click.echo(f" Upgrade with: {finding['upgrade_command']}")
|
|
||||||
|
|
||||||
click.echo(
|
|
||||||
"\nMVT does not install plugin updates. Run the command above when you "
|
|
||||||
"decide to upgrade."
|
|
||||||
)
|
|
||||||
+16
-265
@@ -72,10 +72,6 @@ class Command:
|
|||||||
# down a password to decrypt a backup or flags which are need by some modules.
|
# down a password to decrypt a backup or flags which are need by some modules.
|
||||||
self.module_options = module_options if module_options else {}
|
self.module_options = module_options if module_options else {}
|
||||||
|
|
||||||
# This dictionary maps the modules which were replaced by a module
|
|
||||||
# declaring `replaces` to the module which took their place.
|
|
||||||
self.module_replacements: dict[type[MVTModule], type[MVTModule]] = {}
|
|
||||||
|
|
||||||
# This list will contain all executed modules.
|
# This list will contain all executed modules.
|
||||||
# We can use this to reference e.g. self.executed[0].results.
|
# We can use this to reference e.g. self.executed[0].results.
|
||||||
self.executed: list[MVTModule] = []
|
self.executed: list[MVTModule] = []
|
||||||
@@ -261,174 +257,7 @@ class Command:
|
|||||||
if module not in deduplicated:
|
if module not in deduplicated:
|
||||||
deduplicated.append(module)
|
deduplicated.append(module)
|
||||||
|
|
||||||
available = self._apply_replacements(deduplicated)
|
return deduplicated
|
||||||
self._warn_about_slug_collisions(available)
|
|
||||||
|
|
||||||
return available
|
|
||||||
|
|
||||||
def _warn_about_slug_collisions(self, modules: list[type[MVTModule]]) -> None:
|
|
||||||
"""Report modules writing their results to the same file.
|
|
||||||
|
|
||||||
Results are stored in a file named after the module slug, so two
|
|
||||||
modules sharing one silently overwrite each other. Replacements are
|
|
||||||
already resolved here, so a module deliberately taking over the slug
|
|
||||||
of the module it replaces is not reported: the module it replaced is
|
|
||||||
no longer part of the run.
|
|
||||||
"""
|
|
||||||
modules_by_slug: dict[str, type[MVTModule]] = {}
|
|
||||||
for module in modules:
|
|
||||||
slug = module.get_slug()
|
|
||||||
first = modules_by_slug.setdefault(slug, module)
|
|
||||||
if first is module:
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
"Modules %s from %s and %s from %s both use the slug %s. If "
|
|
||||||
"both run, whichever runs last overwrites the results of the "
|
|
||||||
"other in %s.json.",
|
|
||||||
first.__name__,
|
|
||||||
get_module_origin(first).label,
|
|
||||||
module.__name__,
|
|
||||||
get_module_origin(module).label,
|
|
||||||
slug,
|
|
||||||
slug,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _declared_replacements(
|
|
||||||
self, modules: list[type[MVTModule]]
|
|
||||||
) -> dict[type[MVTModule], type[MVTModule]]:
|
|
||||||
"""Return the replacements declared by the given modules."""
|
|
||||||
replacements: dict[type[MVTModule], type[MVTModule]] = {}
|
|
||||||
for module in modules:
|
|
||||||
replaced = module.replaces
|
|
||||||
if replaced is None or replaced is module:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not module.enabled:
|
|
||||||
# Replacing a module must not disable it, as a disabled
|
|
||||||
# replacement never runs in its place.
|
|
||||||
self.log.debug(
|
|
||||||
"Module %s is disabled and does not replace module %s.",
|
|
||||||
module.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if replaced not in modules:
|
|
||||||
# A module can support several commands while the module it
|
|
||||||
# replaces is only available in some of them.
|
|
||||||
self.log.debug(
|
|
||||||
"Module %s replaces module %s, which is not available "
|
|
||||||
"for the %s command.",
|
|
||||||
module.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
self.name,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if replaced in replacements:
|
|
||||||
self.log.warning(
|
|
||||||
"Modules %s and %s both replace module %s. Both of them "
|
|
||||||
"will run, %s will not, and modules depending on %s will "
|
|
||||||
"use the results of %s. Replacements which share the slug "
|
|
||||||
"of %s overwrite each other's results file.",
|
|
||||||
replacements[replaced].__name__,
|
|
||||||
module.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
replacements[replaced].__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
replacements[replaced] = module
|
|
||||||
|
|
||||||
return replacements
|
|
||||||
|
|
||||||
def _drop_replacement_cycles(
|
|
||||||
self, replacements: dict[type[MVTModule], type[MVTModule]]
|
|
||||||
) -> None:
|
|
||||||
"""Undo the replacements between modules which replace each other."""
|
|
||||||
cyclic: set[type[MVTModule]] = set()
|
|
||||||
for replaced in replacements:
|
|
||||||
walked: list[type[MVTModule]] = []
|
|
||||||
module = replaced
|
|
||||||
while module in replacements and module not in cyclic:
|
|
||||||
if module in walked:
|
|
||||||
cyclic.update(walked[walked.index(module) :])
|
|
||||||
break
|
|
||||||
walked.append(module)
|
|
||||||
module = replacements[module]
|
|
||||||
|
|
||||||
if not cyclic:
|
|
||||||
return
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
"Modules %s replace each other in a cycle. None of them replaces "
|
|
||||||
"anything and all of them will run.",
|
|
||||||
", ".join(sorted(module.__name__ for module in cyclic)),
|
|
||||||
)
|
|
||||||
for module in cyclic:
|
|
||||||
replacements.pop(module, None)
|
|
||||||
|
|
||||||
def _log_replacement(
|
|
||||||
self,
|
|
||||||
replaced: type[MVTModule],
|
|
||||||
module: type[MVTModule],
|
|
||||||
replacement: type[MVTModule],
|
|
||||||
) -> None:
|
|
||||||
"""Report an applied replacement, and any problem with it."""
|
|
||||||
if not issubclass(module, replaced):
|
|
||||||
self.log.warning(
|
|
||||||
"Module %s replaces module %s but is not a subclass of it. "
|
|
||||||
"Its results might not be compatible with what modules "
|
|
||||||
"depending on %s expect.",
|
|
||||||
module.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
)
|
|
||||||
|
|
||||||
if replaced in module.dependencies:
|
|
||||||
self.log.warning(
|
|
||||||
"Module %s depends on module %s, which it also replaces. The "
|
|
||||||
"dependency cannot be satisfied: a replacement has to produce "
|
|
||||||
"that data itself.",
|
|
||||||
module.__name__,
|
|
||||||
replaced.__name__,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.info(
|
|
||||||
"Module %s from %s replaces module %s from %s.",
|
|
||||||
replacement.__name__,
|
|
||||||
get_module_origin(replacement).label,
|
|
||||||
replaced.__name__,
|
|
||||||
get_module_origin(replaced).label,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _apply_replacements(
|
|
||||||
self, modules: list[type[MVTModule]]
|
|
||||||
) -> list[type[MVTModule]]:
|
|
||||||
"""Drop the modules superseded by a module declaring `replaces`."""
|
|
||||||
declared = self._declared_replacements(modules)
|
|
||||||
self._drop_replacement_cycles(declared)
|
|
||||||
|
|
||||||
replacements: dict[type[MVTModule], type[MVTModule]] = {}
|
|
||||||
for replaced, module in declared.items():
|
|
||||||
# Follow chains of replacements, so that a dependency on a
|
|
||||||
# replaced module always resolves to a module which is part of
|
|
||||||
# the run.
|
|
||||||
replacement = module
|
|
||||||
while replacement in declared:
|
|
||||||
replacement = declared[replacement]
|
|
||||||
|
|
||||||
# Only the replacements which are applied are reported, so that
|
|
||||||
# the record matches the modules which actually run.
|
|
||||||
self._log_replacement(replaced, module, replacement)
|
|
||||||
replacements[replaced] = replacement
|
|
||||||
|
|
||||||
self.module_replacements = replacements
|
|
||||||
return [module for module in modules if module not in replacements]
|
|
||||||
|
|
||||||
def init(self) -> None:
|
def init(self) -> None:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
@@ -488,80 +317,6 @@ class Command:
|
|||||||
console.print("")
|
console.print("")
|
||||||
console.print(panel)
|
console.print(panel)
|
||||||
|
|
||||||
def _module_dependencies(
|
|
||||||
self, module: type[MVTModule]
|
|
||||||
) -> list[tuple[type[MVTModule], type[MVTModule]]]:
|
|
||||||
"""Return the (declared, resolved) dependencies of a module.
|
|
||||||
|
|
||||||
A dependency on a module which was replaced is resolved to the module
|
|
||||||
which took its place. A module which replaces one of its own
|
|
||||||
dependencies is not made to depend on itself, while a module which
|
|
||||||
declares itself as a dependency is left alone and still fails the
|
|
||||||
circular dependency check.
|
|
||||||
"""
|
|
||||||
dependencies = []
|
|
||||||
for dependency in module.dependencies:
|
|
||||||
resolved = self.module_replacements.get(dependency, dependency)
|
|
||||||
if resolved is module and dependency is not module:
|
|
||||||
continue
|
|
||||||
dependencies.append((dependency, resolved))
|
|
||||||
|
|
||||||
return dependencies
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _dependency_name(
|
|
||||||
declared: type[MVTModule], resolved: type[MVTModule]
|
|
||||||
) -> str:
|
|
||||||
"""Return how a dependency is named in the messages about it.
|
|
||||||
|
|
||||||
A dependency is named as the module which declared it wrote it, and,
|
|
||||||
when that module was replaced, as the module which runs in its place.
|
|
||||||
"""
|
|
||||||
if declared is resolved:
|
|
||||||
return declared.__name__
|
|
||||||
|
|
||||||
return f"{resolved.__name__} (replacing module {declared.__name__})"
|
|
||||||
|
|
||||||
def _selected_modules(
|
|
||||||
self, modules: list[type[MVTModule]]
|
|
||||||
) -> Optional[list[type[MVTModule]]]:
|
|
||||||
"""Return the modules explicitly requested, or all the enabled ones.
|
|
||||||
|
|
||||||
Returns None when a module was requested by name and no module of
|
|
||||||
that name can be run.
|
|
||||||
"""
|
|
||||||
if not self.module_name:
|
|
||||||
return [module for module in modules if module.enabled]
|
|
||||||
|
|
||||||
selected = [
|
|
||||||
module for module in modules if module.__name__ == self.module_name
|
|
||||||
]
|
|
||||||
|
|
||||||
# A module replacing another one does not have to keep its name, so
|
|
||||||
# the name of a replaced module selects its replacement.
|
|
||||||
if not selected:
|
|
||||||
for replaced, replacement in self.module_replacements.items():
|
|
||||||
if replaced.__name__ != self.module_name or replacement in selected:
|
|
||||||
continue
|
|
||||||
self.log.info(
|
|
||||||
"Module %s was replaced by module %s, which is run "
|
|
||||||
"in its place.",
|
|
||||||
replaced.__name__,
|
|
||||||
replacement.__name__,
|
|
||||||
)
|
|
||||||
selected.append(replacement)
|
|
||||||
|
|
||||||
if not selected:
|
|
||||||
self.log.warning(
|
|
||||||
"No module named %s is available for the %s command. "
|
|
||||||
"No modules will be run.",
|
|
||||||
self.module_name,
|
|
||||||
self.name,
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
return selected
|
|
||||||
|
|
||||||
def _skipped_modules(
|
def _skipped_modules(
|
||||||
self,
|
self,
|
||||||
required: list[type[MVTModule]],
|
required: list[type[MVTModule]],
|
||||||
@@ -594,18 +349,15 @@ class Command:
|
|||||||
if module in skipped:
|
if module in skipped:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for declared, dependency in self._module_dependencies(module):
|
for dependency in module.dependencies:
|
||||||
if dependency not in module_indexes:
|
if dependency not in module_indexes:
|
||||||
# A replaced dependency always resolves to a module of
|
skipped[module] = (module, dependency)
|
||||||
# this command, so an unavailable one is always the
|
|
||||||
# module class the author declared.
|
|
||||||
skipped[module] = (module, declared)
|
|
||||||
changed = True
|
changed = True
|
||||||
self.log.warning(
|
self.log.warning(
|
||||||
"Module %s will be SKIPPED: it depends on module "
|
"Module %s will be SKIPPED: it depends on module "
|
||||||
"%s, which is not available in this command.%s",
|
"%s, which is not available in this command.%s",
|
||||||
module.__name__,
|
module.__name__,
|
||||||
declared.__name__,
|
dependency.__name__,
|
||||||
remainder,
|
remainder,
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
@@ -620,7 +372,7 @@ class Command:
|
|||||||
"module %s, itself skipped for depending on "
|
"module %s, itself skipped for depending on "
|
||||||
"unavailable module %s.%s",
|
"unavailable module %s.%s",
|
||||||
module.__name__,
|
module.__name__,
|
||||||
self._dependency_name(declared, dependency),
|
dependency.__name__,
|
||||||
missing.__name__,
|
missing.__name__,
|
||||||
remainder,
|
remainder,
|
||||||
)
|
)
|
||||||
@@ -631,7 +383,7 @@ class Command:
|
|||||||
"module %s, which depends on unavailable "
|
"module %s, which depends on unavailable "
|
||||||
"module %s.%s",
|
"module %s.%s",
|
||||||
module.__name__,
|
module.__name__,
|
||||||
self._dependency_name(declared, dependency),
|
dependency.__name__,
|
||||||
root.__name__,
|
root.__name__,
|
||||||
missing.__name__,
|
missing.__name__,
|
||||||
remainder,
|
remainder,
|
||||||
@@ -645,9 +397,12 @@ class Command:
|
|||||||
modules = self._available_modules()
|
modules = self._available_modules()
|
||||||
module_indexes = {module: index for index, module in enumerate(modules)}
|
module_indexes = {module: index for index, module in enumerate(modules)}
|
||||||
|
|
||||||
selected = self._selected_modules(modules)
|
if self.module_name:
|
||||||
if selected is None:
|
selected = [
|
||||||
return None
|
module for module in modules if module.__name__ == self.module_name
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
selected = [module for module in modules if module.enabled]
|
||||||
|
|
||||||
required: set[type[MVTModule]] = set()
|
required: set[type[MVTModule]] = set()
|
||||||
pending = list(selected)
|
pending = list(selected)
|
||||||
@@ -656,7 +411,7 @@ class Command:
|
|||||||
if module in required:
|
if module in required:
|
||||||
continue
|
continue
|
||||||
required.add(module)
|
required.add(module)
|
||||||
for _, dependency in self._module_dependencies(module):
|
for dependency in module.dependencies:
|
||||||
# Unavailable dependencies are reported by _skipped_modules().
|
# Unavailable dependencies are reported by _skipped_modules().
|
||||||
if dependency in module_indexes:
|
if dependency in module_indexes:
|
||||||
pending.append(dependency)
|
pending.append(dependency)
|
||||||
@@ -669,14 +424,13 @@ class Command:
|
|||||||
"Every selected module was skipped for an unavailable "
|
"Every selected module was skipped for an unavailable "
|
||||||
"dependency. No modules will be run."
|
"dependency. No modules will be run."
|
||||||
)
|
)
|
||||||
return None
|
|
||||||
|
|
||||||
dependents: dict[type[MVTModule], list[type[MVTModule]]] = {
|
dependents: dict[type[MVTModule], list[type[MVTModule]]] = {
|
||||||
module: [] for module in runnable
|
module: [] for module in runnable
|
||||||
}
|
}
|
||||||
indegree = {module: 0 for module in runnable}
|
indegree = {module: 0 for module in runnable}
|
||||||
for module in runnable:
|
for module in runnable:
|
||||||
for _, dependency in self._module_dependencies(module):
|
for dependency in module.dependencies:
|
||||||
if dependency not in indegree:
|
if dependency not in indegree:
|
||||||
continue
|
continue
|
||||||
dependents[dependency].append(module)
|
dependents[dependency].append(module)
|
||||||
@@ -732,12 +486,9 @@ class Command:
|
|||||||
module_options=self.module_options,
|
module_options=self.module_options,
|
||||||
log=module_logger,
|
log=module_logger,
|
||||||
)
|
)
|
||||||
# Dependencies are keyed by the module class they declare, even
|
|
||||||
# when it was replaced, so that a module asking for the results
|
|
||||||
# of a replaced module receives those of its replacement.
|
|
||||||
m.dependency_modules = {
|
m.dependency_modules = {
|
||||||
dependency: executed_by_type[resolved]
|
dependency: executed_by_type[dependency]
|
||||||
for dependency, resolved in self._module_dependencies(module)
|
for dependency in module.dependencies
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.iocs.total_ioc_count:
|
if self.iocs.total_ioc_count:
|
||||||
|
|||||||
@@ -9,83 +9,34 @@ import shlex
|
|||||||
import click
|
import click
|
||||||
from click.shell_completion import get_completion_class
|
from click.shell_completion import get_completion_class
|
||||||
|
|
||||||
from .help import HELP_MSG_COMPLETION
|
|
||||||
|
|
||||||
|
|
||||||
SUPPORTED_SHELLS = ("bash", "zsh", "fish")
|
SUPPORTED_SHELLS = ("bash", "zsh", "fish")
|
||||||
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
|
|
||||||
|
|
||||||
COMPLETION_INSTRUCTIONS = """Shell completion for mvt, mvt-ios and mvt-android
|
|
||||||
|
|
||||||
Print one completion script covering the three commands:
|
def completion_instructions(program_name: str) -> str:
|
||||||
mvt completion bash > ~/.mvt-complete.bash
|
return f"""Shell completion for {program_name}
|
||||||
mvt completion zsh > ~/.mvt-complete.zsh
|
|
||||||
mkdir -p ~/.config/fish/conf.d
|
Print a completion script:
|
||||||
mvt completion fish > ~/.config/fish/conf.d/mvt-completion.fish
|
{program_name} completion bash > ~/.{program_name}-complete.bash
|
||||||
|
{program_name} completion zsh > ~/.{program_name}-complete.zsh
|
||||||
|
mkdir -p ~/.config/fish/completions
|
||||||
|
{program_name} completion fish > ~/.config/fish/completions/{program_name}.fish
|
||||||
|
|
||||||
Load the generated Bash script from ~/.bashrc:
|
Load the generated Bash script from ~/.bashrc:
|
||||||
[ -f ~/.mvt-complete.bash ] && . ~/.mvt-complete.bash
|
[ -f ~/.{program_name}-complete.bash ] && . ~/.{program_name}-complete.bash
|
||||||
|
|
||||||
Load the generated Zsh script from ~/.zshrc:
|
Load the generated Zsh script from ~/.zshrc:
|
||||||
[ -f ~/.mvt-complete.zsh ] && . ~/.mvt-complete.zsh
|
[ -f ~/.{program_name}-complete.zsh ] && . ~/.{program_name}-complete.zsh
|
||||||
|
|
||||||
Fish loads the files in ~/.config/fish/conf.d automatically.
|
Fish loads completion files from ~/.config/fish/completions automatically.
|
||||||
|
|
||||||
To write these files and update the Bash/Zsh shell configuration automatically:
|
To write these files and update Bash/Zsh shell configuration automatically:
|
||||||
mvt completion bash --install
|
{program_name} completion bash --install
|
||||||
mvt completion zsh --install
|
{program_name} completion zsh --install
|
||||||
mvt completion fish --install
|
{program_name} completion fish --install
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def _mvt_programs() -> list[tuple[str, click.Command]]:
|
|
||||||
"""Return the console script name and CLI group of every MVT program.
|
|
||||||
|
|
||||||
The three CLIs are imported here rather than at module level: mvt.cli
|
|
||||||
imports this module while it is being defined, and generating a completion
|
|
||||||
script should not make the start-up of `mvt` import the platform CLIs.
|
|
||||||
"""
|
|
||||||
from mvt.android.cli import cli as android_cli
|
|
||||||
from mvt.cli import cli as mvt_cli
|
|
||||||
from mvt.ios.cli import cli as ios_cli
|
|
||||||
|
|
||||||
return [("mvt", mvt_cli), ("mvt-ios", ios_cli), ("mvt-android", android_cli)]
|
|
||||||
|
|
||||||
|
|
||||||
@click.command(
|
|
||||||
"completion",
|
|
||||||
context_settings=CONTEXT_SETTINGS,
|
|
||||||
help=HELP_MSG_COMPLETION,
|
|
||||||
short_help="Generate or install shell completion",
|
|
||||||
)
|
|
||||||
@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS))
|
|
||||||
@click.option(
|
|
||||||
"--install",
|
|
||||||
is_flag=True,
|
|
||||||
help="Write completion files and update shell configuration.",
|
|
||||||
)
|
|
||||||
def completion(shell, install):
|
|
||||||
if shell is None:
|
|
||||||
if install:
|
|
||||||
raise click.UsageError("A shell is required when using --install.")
|
|
||||||
click.echo(COMPLETION_INSTRUCTIONS)
|
|
||||||
return
|
|
||||||
|
|
||||||
if install:
|
|
||||||
script_path = install_completion_script(shell)
|
|
||||||
click.echo(
|
|
||||||
f"Installed {shell} completion for mvt, mvt-ios and mvt-android "
|
|
||||||
f"to {script_path}"
|
|
||||||
)
|
|
||||||
if shell in ("bash", "zsh"):
|
|
||||||
click.echo(f"Updated ~/.{shell}rc")
|
|
||||||
else:
|
|
||||||
click.echo("Fish loads the files in ~/.config/fish/conf.d automatically.")
|
|
||||||
return
|
|
||||||
|
|
||||||
click.echo(generate_mvt_completion_script(shell))
|
|
||||||
|
|
||||||
|
|
||||||
def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str:
|
def generate_completion_script(cli: click.Command, program_name: str, shell: str) -> str:
|
||||||
completion_class = get_completion_class(shell)
|
completion_class = get_completion_class(shell)
|
||||||
if completion_class is None:
|
if completion_class is None:
|
||||||
@@ -95,49 +46,41 @@ def generate_completion_script(cli: click.Command, program_name: str, shell: str
|
|||||||
return completion_class(cli, {}, program_name, complete_var).source()
|
return completion_class(cli, {}, program_name, complete_var).source()
|
||||||
|
|
||||||
|
|
||||||
def generate_mvt_completion_script(shell: str) -> str:
|
def install_completion_script(
|
||||||
"""Return one script completing every MVT command.
|
cli: click.Command,
|
||||||
|
program_name: str,
|
||||||
Click names the completion function of each program after the program, so
|
shell: str,
|
||||||
the scripts of the three commands can simply be concatenated.
|
) -> Path:
|
||||||
"""
|
script = generate_completion_script(cli, program_name, shell)
|
||||||
scripts = [
|
script_path = _completion_script_path(program_name, shell)
|
||||||
generate_completion_script(cli, program_name, shell).strip("\n")
|
|
||||||
for program_name, cli in _mvt_programs()
|
|
||||||
]
|
|
||||||
return "\n\n".join(scripts)
|
|
||||||
|
|
||||||
|
|
||||||
def install_completion_script(shell: str) -> Path:
|
|
||||||
script = generate_mvt_completion_script(shell)
|
|
||||||
script_path = _completion_script_path(shell)
|
|
||||||
script_path.parent.mkdir(parents=True, exist_ok=True)
|
script_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
script_path.write_text(f"{script}\n", encoding="utf-8")
|
script_path.write_text(script, encoding="utf-8")
|
||||||
|
|
||||||
if shell in ("bash", "zsh"):
|
if shell in ("bash", "zsh"):
|
||||||
_install_shell_source_line(shell, script_path)
|
_install_shell_source_line(program_name, shell, script_path)
|
||||||
|
|
||||||
return script_path
|
return script_path
|
||||||
|
|
||||||
|
|
||||||
def _completion_script_path(shell: str) -> Path:
|
def _completion_script_path(program_name: str, shell: str) -> Path:
|
||||||
home = Path.home()
|
home = Path.home()
|
||||||
|
|
||||||
if shell == "fish":
|
if shell == "fish":
|
||||||
# conf.d is sourced when the shell starts, unlike the completions
|
return home / ".config" / "fish" / "completions" / f"{program_name}.fish"
|
||||||
# folder, whose files fish loads on demand by command name.
|
|
||||||
return home / ".config" / "fish" / "conf.d" / "mvt-completion.fish"
|
|
||||||
|
|
||||||
return home / f".mvt-complete.{shell}"
|
return home / f".{program_name}-complete.{shell}"
|
||||||
|
|
||||||
|
|
||||||
def _install_shell_source_line(shell: str, script_path: Path) -> None:
|
def _install_shell_source_line(program_name: str, shell: str, script_path: Path) -> None:
|
||||||
shell_config_path = Path.home() / f".{shell}rc"
|
shell_config_path = Path.home() / f".{shell}rc"
|
||||||
source_line = (
|
source_line = (
|
||||||
f"[ -f {shlex.quote(str(script_path))} ] && "
|
f"[ -f {shlex.quote(str(script_path))} ] && "
|
||||||
f". {shlex.quote(str(script_path))}"
|
f". {shlex.quote(str(script_path))}"
|
||||||
)
|
)
|
||||||
block = f"# MVT shell completion\n{source_line}\n"
|
block = (
|
||||||
|
f"# MVT shell completion for {program_name}\n"
|
||||||
|
f"{source_line}\n"
|
||||||
|
)
|
||||||
|
|
||||||
if shell_config_path.exists():
|
if shell_config_path.exists():
|
||||||
shell_config = shell_config_path.read_text(encoding="utf-8")
|
shell_config = shell_config_path.read_text(encoding="utf-8")
|
||||||
|
|||||||
+1
-13
@@ -17,29 +17,17 @@ HELP_MSG_LOAD_MODULE = (
|
|||||||
HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing"
|
HELP_MSG_NONINTERACTIVE = "Don't ask interactive questions during processing"
|
||||||
HELP_MSG_HASHES = "Generate hashes of all the files analyzed"
|
HELP_MSG_HASHES = "Generate hashes of all the files analyzed"
|
||||||
HELP_MSG_VERBOSE = "Verbose mode"
|
HELP_MSG_VERBOSE = "Verbose mode"
|
||||||
HELP_MSG_VERBOSE_COMMAND = (
|
|
||||||
"Verbose mode (kept for compatibility, pass --verbose before the command "
|
|
||||||
"name instead)"
|
|
||||||
)
|
|
||||||
HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators"
|
HELP_MSG_CHECK_IOCS = "Compare stored JSON results to provided indicators"
|
||||||
HELP_MSG_STIX2 = "Download public STIX2 indicators"
|
HELP_MSG_STIX2 = "Download public STIX2 indicators"
|
||||||
HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check"
|
HELP_MSG_DISABLE_UPDATE_CHECK = "Disable MVT version update check"
|
||||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check"
|
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK = "Disable indicators update check"
|
||||||
HELP_MSG_COMPLETION = (
|
HELP_MSG_COMPLETION = "Generate or install shell completion"
|
||||||
"Generate or install shell completion for mvt, mvt-ios and mvt-android"
|
|
||||||
)
|
|
||||||
HELP_MSG_PLUGINS = "Inspect the installed MVT plugin packages"
|
|
||||||
HELP_MSG_PLUGINS_LIST = "List the installed plugins and what they contribute to MVT"
|
|
||||||
HELP_MSG_PLUGINS_CHECK_UPDATES = (
|
|
||||||
"Check the installed plugins for updates without installing them"
|
|
||||||
)
|
|
||||||
|
|
||||||
# IOS Specific
|
# IOS Specific
|
||||||
HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup"
|
HELP_MSG_DECRYPT_BACKUP = "Decrypt an encrypted iTunes backup"
|
||||||
HELP_MSG_BACKUP_DESTINATION = (
|
HELP_MSG_BACKUP_DESTINATION = (
|
||||||
"Path to the folder where the decrypted backup should be stored"
|
"Path to the folder where the decrypted backup should be stored"
|
||||||
)
|
)
|
||||||
HELP_MSG_DECRYPT_JOBS = "Number of files to decrypt concurrently"
|
|
||||||
HELP_MSG_IOS_BACKUP_PASSWORD = (
|
HELP_MSG_IOS_BACKUP_PASSWORD = (
|
||||||
"Password to use to decrypt the backup (or, set the {MVT_IOS_BACKUP_PASSWORD} "
|
"Password to use to decrypt the backup (or, set the {MVT_IOS_BACKUP_PASSWORD} "
|
||||||
"environment variable)"
|
"environment variable)"
|
||||||
|
|||||||
+62
-125
@@ -8,125 +8,10 @@ import logging
|
|||||||
import requests
|
import requests
|
||||||
from rich import print as rich_print
|
from rich import print as rich_print
|
||||||
|
|
||||||
from .config import settings
|
from .updates import IndicatorsUpdates, MVTUpdates
|
||||||
from .updates import (
|
|
||||||
IndicatorsUpdates,
|
|
||||||
MVTUpdates,
|
|
||||||
PluginUpdates,
|
|
||||||
installed_plugin_distributions,
|
|
||||||
)
|
|
||||||
from .version import MVT_VERSION
|
from .version import MVT_VERSION
|
||||||
|
|
||||||
|
|
||||||
def _check_version_updates(log: logging.Logger) -> None:
|
|
||||||
try:
|
|
||||||
mvt_updates = MVTUpdates()
|
|
||||||
latest_version = mvt_updates.check()
|
|
||||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
|
||||||
rich_print(
|
|
||||||
"\t[bold]Note: Could not check for MVT updates.[/bold] "
|
|
||||||
"You may be working offline. Please update MVT regularly."
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
log.error("Error encountered when trying to check latest MVT version: %s", e)
|
|
||||||
else:
|
|
||||||
if latest_version:
|
|
||||||
rich_print(
|
|
||||||
f"\t[bold]Version {latest_version} is available! "
|
|
||||||
"Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_indicator_updates(log: logging.Logger) -> None:
|
|
||||||
ioc_updates = IndicatorsUpdates()
|
|
||||||
|
|
||||||
# Before proceeding, we check if we have downloaded an indicators index.
|
|
||||||
# If not, there's no point in proceeding with the updates check.
|
|
||||||
if ioc_updates.get_latest_update() == 0:
|
|
||||||
rich_print(
|
|
||||||
"\t[bold]You have not yet downloaded any indicators, check "
|
|
||||||
"the `download-iocs` command![/bold]"
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
# We only perform this check at a fixed frequency, in order to not
|
|
||||||
# overburden the user with too many lookups if the command is being run
|
|
||||||
# multiple times.
|
|
||||||
should_check, hours = ioc_updates.should_check()
|
|
||||||
if not should_check:
|
|
||||||
rich_print(
|
|
||||||
f"\tIndicators updates checked recently, next automatic check "
|
|
||||||
f"in {int(hours)} hours"
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
ioc_to_update = ioc_updates.check()
|
|
||||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
|
||||||
rich_print(
|
|
||||||
"\t[bold]Note: Could not check for indicator updates.[/bold] "
|
|
||||||
"You may be working offline. Please update MVT indicators regularly."
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
log.error("Error encountered when trying to check latest MVT indicators: %s", e)
|
|
||||||
else:
|
|
||||||
if ioc_to_update:
|
|
||||||
rich_print(
|
|
||||||
"\t[bold]There are updates to your indicators files! "
|
|
||||||
"Run the `download-iocs` command to update![/bold]"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
rich_print("\tYour indicators files seem to be up to date.")
|
|
||||||
|
|
||||||
|
|
||||||
def _print_plugin_updates(findings: list) -> None:
|
|
||||||
if not findings:
|
|
||||||
return
|
|
||||||
|
|
||||||
rich_print("\t[bold]Plugin updates available:[/bold]")
|
|
||||||
for finding in findings:
|
|
||||||
rich_print(
|
|
||||||
f"\t {finding['name']} {finding['installed']} → "
|
|
||||||
f"{finding['latest']} ({finding['upgrade_command']})"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_plugin_updates(log: logging.Logger) -> None:
|
|
||||||
if not settings.NETWORK_ACCESS_ALLOWED:
|
|
||||||
return
|
|
||||||
|
|
||||||
# This runs on every command, so nothing here, including reading back what
|
|
||||||
# the latest check stored, may ever interrupt MVT.
|
|
||||||
try:
|
|
||||||
distributions = installed_plugin_distributions()
|
|
||||||
|
|
||||||
# There is nothing to check when MVT was not extended with any plugin.
|
|
||||||
if not distributions:
|
|
||||||
return
|
|
||||||
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
|
|
||||||
# We only perform this check at a fixed frequency, in order to not
|
|
||||||
# overburden the user (and the plugin repositories) with too many
|
|
||||||
# lookups. In between checks we print the findings of the latest one,
|
|
||||||
# leaving out those which no longer apply to what is installed.
|
|
||||||
should_check, _ = plugin_updates.should_check()
|
|
||||||
if not should_check:
|
|
||||||
_print_plugin_updates(plugin_updates.current_findings(distributions))
|
|
||||||
return
|
|
||||||
|
|
||||||
findings = plugin_updates.check()
|
|
||||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
|
||||||
rich_print(
|
|
||||||
"\t[bold]Note: Could not check for plugin updates.[/bold] "
|
|
||||||
"You may be working offline. Please update your MVT plugins regularly."
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
log.error("Error encountered when trying to check MVT plugin updates: %s", e)
|
|
||||||
else:
|
|
||||||
_print_plugin_updates(findings)
|
|
||||||
|
|
||||||
|
|
||||||
def check_updates(
|
def check_updates(
|
||||||
disable_version_check: bool = False, disable_indicator_check: bool = False
|
disable_version_check: bool = False, disable_indicator_check: bool = False
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -134,25 +19,77 @@ def check_updates(
|
|||||||
|
|
||||||
# First we check for MVT version updates.
|
# First we check for MVT version updates.
|
||||||
if not disable_version_check:
|
if not disable_version_check:
|
||||||
_check_version_updates(log)
|
try:
|
||||||
|
mvt_updates = MVTUpdates()
|
||||||
|
latest_version = mvt_updates.check()
|
||||||
|
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||||
|
rich_print(
|
||||||
|
"\t\t[bold]Note: Could not check for MVT updates.[/bold] "
|
||||||
|
"You may be working offline. Please update MVT regularly."
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.error(
|
||||||
|
"Error encountered when trying to check latest MVT version: %s", e
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if latest_version:
|
||||||
|
rich_print(
|
||||||
|
f"\t\t[bold]Version {latest_version} is available! "
|
||||||
|
"Upgrade mvt with `pip3 install -U mvt` or with `pipx upgrade mvt`[/bold]"
|
||||||
|
)
|
||||||
|
|
||||||
# Then we check for indicators files updates.
|
# Then we check for indicators files updates.
|
||||||
if not disable_indicator_check:
|
if not disable_indicator_check:
|
||||||
_check_indicator_updates(log)
|
ioc_updates = IndicatorsUpdates()
|
||||||
|
|
||||||
# Finally we check for updates to the installed plugin packages. MVT never
|
# Before proceeding, we check if we have downloaded an indicators index.
|
||||||
# installs an update itself, it only reports the command which does.
|
# If not, there's no point in proceeding with the updates check.
|
||||||
if not disable_version_check:
|
if ioc_updates.get_latest_update() == 0:
|
||||||
_check_plugin_updates(log)
|
rich_print(
|
||||||
|
"\t\t[bold]You have not yet downloaded any indicators, check "
|
||||||
|
"the `download-iocs` command![/bold]"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# We only perform this check at a fixed frequency, in order to not
|
||||||
|
# overburden the user with too many lookups if the command is being run
|
||||||
|
# multiple times.
|
||||||
|
should_check, hours = ioc_updates.should_check()
|
||||||
|
if not should_check:
|
||||||
|
rich_print(
|
||||||
|
f"\t\tIndicators updates checked recently, next automatic check "
|
||||||
|
f"in {int(hours)} hours"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
ioc_to_update = ioc_updates.check()
|
||||||
|
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||||
|
rich_print(
|
||||||
|
"\t\t[bold]Note: Could not check for indicator updates.[/bold] "
|
||||||
|
"You may be working offline. Please update MVT indicators regularly."
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
log.error(
|
||||||
|
"Error encountered when trying to check latest MVT indicators: %s", e
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if ioc_to_update:
|
||||||
|
rich_print(
|
||||||
|
"\t\t[bold]There are updates to your indicators files! "
|
||||||
|
"Run the `download-iocs` command to update![/bold]"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
rich_print("\t\tYour indicators files seem to be up to date.")
|
||||||
|
|
||||||
|
|
||||||
def logo(
|
def logo(
|
||||||
disable_version_check: bool = False, disable_indicator_check: bool = False
|
disable_version_check: bool = False, disable_indicator_check: bool = False
|
||||||
) -> None:
|
) -> None:
|
||||||
rich_print("\n")
|
rich_print("\n")
|
||||||
rich_print("\t[bold]MVT - Mobile Verification Toolkit[/bold]\n")
|
rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
|
||||||
rich_print("\thttps://mvt.re")
|
rich_print("\t\thttps://mvt.re")
|
||||||
rich_print(f"\tVersion: {MVT_VERSION}\n")
|
rich_print(f"\t\tVersion: {MVT_VERSION}")
|
||||||
|
|
||||||
check_updates(disable_version_check, disable_indicator_check)
|
check_updates(disable_version_check, disable_indicator_check)
|
||||||
|
|
||||||
|
|||||||
@@ -46,11 +46,6 @@ class MVTModule:
|
|||||||
slug: Optional[str] = None
|
slug: Optional[str] = None
|
||||||
dependencies: Sequence[type["MVTModule"]] = ()
|
dependencies: Sequence[type["MVTModule"]] = ()
|
||||||
supported_commands: Sequence[tuple[str, str]] = ()
|
supported_commands: Sequence[tuple[str, str]] = ()
|
||||||
# A custom module can name a module class it supersedes, usually a
|
|
||||||
# built-in one. When both are available to a command, the named module is
|
|
||||||
# dropped from the run and this module takes its place, including in the
|
|
||||||
# dependencies of any other module.
|
|
||||||
replaces: Optional[type["MVTModule"]] = None
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import inspect
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import sys
|
import sys
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
@@ -18,7 +17,6 @@ from pathlib import Path
|
|||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
from typing import Iterable, Optional
|
from typing import Iterable, Optional
|
||||||
|
|
||||||
from .cli_plugins import CUSTOM_COMMAND_MODULE_PREFIX
|
|
||||||
from .module import MVTModule
|
from .module import MVTModule
|
||||||
from .version import MVT_VERSION
|
from .version import MVT_VERSION
|
||||||
|
|
||||||
@@ -28,7 +26,6 @@ EXTERNAL_LOGGER_NAMESPACE = "mvt.ext"
|
|||||||
PLUGIN_PACKAGE_PREFIX = "mvt_plugin_"
|
PLUGIN_PACKAGE_PREFIX = "mvt_plugin_"
|
||||||
_ORIGIN_ATTRIBUTE = "_mvt_module_origin"
|
_ORIGIN_ATTRIBUTE = "_mvt_module_origin"
|
||||||
_PATH_MODULE_PREFIX = "_mvt_custom_module_"
|
_PATH_MODULE_PREFIX = "_mvt_custom_module_"
|
||||||
_LOADED_FILE_DIGEST = re.compile(r"_[0-9a-f]{16}$")
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -69,25 +66,6 @@ def _module_name_for_path(path: Path) -> str:
|
|||||||
return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}"
|
return f"{_PATH_MODULE_PREFIX}{path.stem}_{digest}"
|
||||||
|
|
||||||
|
|
||||||
def _is_builtin_logger_name(name: str) -> bool:
|
|
||||||
return name == "mvt" or name.startswith("mvt.")
|
|
||||||
|
|
||||||
|
|
||||||
def _loaded_file_stem(name: str, prefix: str) -> str:
|
|
||||||
"""Recover a loaded file's name from the import name MVT gave it."""
|
|
||||||
return _LOADED_FILE_DIGEST.sub("", name[len(prefix) :])
|
|
||||||
|
|
||||||
|
|
||||||
def _external_logger_name(name: str) -> str:
|
|
||||||
"""Return the "mvt.ext" logger name external code logs under."""
|
|
||||||
top_level, separator, rest = name.partition(".")
|
|
||||||
if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len(
|
|
||||||
PLUGIN_PACKAGE_PREFIX
|
|
||||||
):
|
|
||||||
name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest
|
|
||||||
return f"{EXTERNAL_LOGGER_NAMESPACE}.{name}"
|
|
||||||
|
|
||||||
|
|
||||||
def get_module_logger(module_class: type[MVTModule]) -> logging.Logger:
|
def get_module_logger(module_class: type[MVTModule]) -> logging.Logger:
|
||||||
"""Return the logger a module's records should be emitted through.
|
"""Return the logger a module's records should be emitted through.
|
||||||
|
|
||||||
@@ -102,35 +80,19 @@ def get_module_logger(module_class: type[MVTModule]) -> logging.Logger:
|
|||||||
"mvt_plugin_<name>" naming convention log under "mvt.ext.<name>".
|
"mvt_plugin_<name>" naming convention log under "mvt.ext.<name>".
|
||||||
"""
|
"""
|
||||||
name = module_class.__module__
|
name = module_class.__module__
|
||||||
if _is_builtin_logger_name(name):
|
if name == "mvt" or name.startswith("mvt."):
|
||||||
return logging.getLogger(name)
|
return logging.getLogger(name)
|
||||||
|
|
||||||
if name.startswith(_PATH_MODULE_PREFIX):
|
if name.startswith(_PATH_MODULE_PREFIX):
|
||||||
file_name = Path(get_module_origin(module_class).name).stem
|
name = Path(get_module_origin(module_class).name).stem
|
||||||
return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{file_name}")
|
else:
|
||||||
|
top_level, separator, rest = name.partition(".")
|
||||||
|
if top_level.startswith(PLUGIN_PACKAGE_PREFIX) and len(top_level) > len(
|
||||||
|
PLUGIN_PACKAGE_PREFIX
|
||||||
|
):
|
||||||
|
name = top_level[len(PLUGIN_PACKAGE_PREFIX) :] + separator + rest
|
||||||
|
|
||||||
return logging.getLogger(_external_logger_name(name))
|
return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{name}")
|
||||||
|
|
||||||
|
|
||||||
def get_plugin_logger(name: str) -> logging.Logger:
|
|
||||||
"""Return a general logger for use in custom MVT plugins.
|
|
||||||
|
|
||||||
Call it with ``__name__``. The logger sits under "mvt.ext". That is
|
|
||||||
where get_module_logger() puts module classes. A file loaded with
|
|
||||||
--load-module or --load-command is named after the file.
|
|
||||||
"""
|
|
||||||
if _is_builtin_logger_name(name):
|
|
||||||
return logging.getLogger(name)
|
|
||||||
|
|
||||||
# A file loaded with --load-module or --load-command is imported under a
|
|
||||||
# mangled name. Log it under the file it came from. get_module_logger()
|
|
||||||
# does the same for the module classes such a file defines.
|
|
||||||
for prefix in (_PATH_MODULE_PREFIX, CUSTOM_COMMAND_MODULE_PREFIX):
|
|
||||||
if name.startswith(prefix):
|
|
||||||
stem = _loaded_file_stem(name, prefix)
|
|
||||||
return logging.getLogger(f"{EXTERNAL_LOGGER_NAMESPACE}.{stem}")
|
|
||||||
|
|
||||||
return logging.getLogger(_external_logger_name(name))
|
|
||||||
|
|
||||||
|
|
||||||
def _iter_module_files(path: Path) -> Iterable[Path]:
|
def _iter_module_files(path: Path) -> Iterable[Path]:
|
||||||
@@ -217,36 +179,21 @@ def _module_key(module_class: type[MVTModule]) -> tuple[str, str]:
|
|||||||
return (source, module_class.__qualname__)
|
return (source, module_class.__qualname__)
|
||||||
|
|
||||||
|
|
||||||
def distribution_direct_url(dist: importlib.metadata.Distribution) -> Optional[dict]:
|
|
||||||
"""Return the PEP 610 direct URL metadata of a distribution, if recorded.
|
|
||||||
|
|
||||||
Packages installed from an index have no direct URL metadata, while
|
|
||||||
packages installed directly from a repository or from a local folder
|
|
||||||
record where they were installed from in ``direct_url.json``.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
direct_url_text = dist.read_text("direct_url.json")
|
|
||||||
if not direct_url_text:
|
|
||||||
return None
|
|
||||||
direct_url = json.loads(direct_url_text)
|
|
||||||
return direct_url if isinstance(direct_url, dict) else None
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]:
|
def _distribution_commit(dist: importlib.metadata.Distribution) -> Optional[str]:
|
||||||
"""Return the VCS commit a distribution was installed from, if recorded.
|
"""Return the VCS commit a distribution was installed from, if recorded.
|
||||||
|
|
||||||
Packages installed directly from a repository (``pip install git+...``)
|
Packages installed directly from a repository (``pip install git+...``)
|
||||||
record the commit in ``direct_url.json`` (PEP 610).
|
record the commit in ``direct_url.json`` (PEP 610).
|
||||||
"""
|
"""
|
||||||
vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info")
|
try:
|
||||||
if not isinstance(vcs_info, dict):
|
direct_url_text = dist.read_text("direct_url.json")
|
||||||
|
if not direct_url_text:
|
||||||
|
return None
|
||||||
|
commit = json.loads(direct_url_text).get("vcs_info", {}).get("commit_id")
|
||||||
|
return commit if isinstance(commit, str) else None
|
||||||
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
commit = vcs_info.get("commit_id")
|
|
||||||
return commit if isinstance(commit, str) else None
|
|
||||||
|
|
||||||
|
|
||||||
def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin:
|
def _entry_point_origin(entry_point: importlib.metadata.EntryPoint) -> ModuleOrigin:
|
||||||
name = entry_point.name
|
name = entry_point.name
|
||||||
@@ -401,14 +348,4 @@ def module_supports_command(
|
|||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
pairs = {tuple(entry) for entry in supported_commands}
|
return (platform, command) in {tuple(entry) for entry in supported_commands}
|
||||||
if (platform, command) in pairs:
|
|
||||||
return True
|
|
||||||
|
|
||||||
# A module which implements check_indicators() is re-checked by check-iocs
|
|
||||||
# for its platform. It does not need to declare the check-iocs pair.
|
|
||||||
return (
|
|
||||||
command == "check-iocs"
|
|
||||||
and platform in {entry[0] for entry in pairs if entry}
|
|
||||||
and module_class.check_indicators is not MVTModule.check_indicators
|
|
||||||
)
|
|
||||||
+3
-416
@@ -3,13 +3,8 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import importlib.metadata
|
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shlex
|
|
||||||
import subprocess
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
|
|
||||||
@@ -17,35 +12,14 @@ import requests
|
|||||||
import yaml
|
import yaml
|
||||||
from packaging import version
|
from packaging import version
|
||||||
|
|
||||||
from .cli_plugins import (
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
)
|
|
||||||
from .config import settings
|
from .config import settings
|
||||||
from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER
|
from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER
|
||||||
from .module_loader import MODULES_ENTRY_POINT_GROUP, distribution_direct_url
|
|
||||||
from .version import MVT_VERSION
|
from .version import MVT_VERSION
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
# In hours.
|
# In hours.
|
||||||
INDICATORS_CHECK_FREQUENCY = 12
|
INDICATORS_CHECK_FREQUENCY = 12
|
||||||
PLUGINS_CHECK_FREQUENCY = 12
|
|
||||||
|
|
||||||
# The entry-point groups a package can use to extend MVT.
|
|
||||||
PLUGIN_ENTRY_POINT_GROUPS = (
|
|
||||||
MODULES_ENTRY_POINT_GROUP,
|
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
)
|
|
||||||
SHORT_COMMIT_LENGTH = 8
|
|
||||||
# The keys every cached finding has to carry to be printed.
|
|
||||||
FINDING_KEYS = ("name", "installed", "latest", "upgrade_command")
|
|
||||||
# Options which stop ssh from waiting for an answer nobody is there to give.
|
|
||||||
_SSH_BATCH_OPTIONS = ("-o", "BatchMode=yes", "-o", "ConnectTimeout=10")
|
|
||||||
_COMMIT_PATTERN = re.compile(r"\A[0-9a-f]{7,40}\Z")
|
|
||||||
|
|
||||||
|
|
||||||
class MVTUpdates:
|
class MVTUpdates:
|
||||||
@@ -64,395 +38,6 @@ class MVTUpdates:
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def installed_plugin_distributions() -> list[importlib.metadata.Distribution]:
|
|
||||||
"""Return the installed distributions which extend MVT.
|
|
||||||
|
|
||||||
A plugin package is any distribution registering at least one entry point
|
|
||||||
in the module or CLI command groups. Distributions are returned once each,
|
|
||||||
sorted by name. MVT itself is not a plugin and is never returned.
|
|
||||||
"""
|
|
||||||
distributions: dict[str, importlib.metadata.Distribution] = {}
|
|
||||||
|
|
||||||
for group in PLUGIN_ENTRY_POINT_GROUPS:
|
|
||||||
try:
|
|
||||||
entry_points = importlib.metadata.entry_points(group=group)
|
|
||||||
except Exception as exc:
|
|
||||||
log.warning(
|
|
||||||
"Unable to discover installed plugin packages in entry-point "
|
|
||||||
"group %s: %s",
|
|
||||||
group,
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
for entry_point in entry_points:
|
|
||||||
# Manually constructed entry points have no associated distribution.
|
|
||||||
dist = getattr(entry_point, "dist", None)
|
|
||||||
if dist is None:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
name = dist.name
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if not name or name == "mvt":
|
|
||||||
continue
|
|
||||||
distributions.setdefault(name, dist)
|
|
||||||
|
|
||||||
return [distributions[name] for name in sorted(distributions)]
|
|
||||||
|
|
||||||
|
|
||||||
def _is_usable_finding(finding: object) -> bool:
|
|
||||||
"""Check that a cached finding carries everything needed to print it."""
|
|
||||||
if not isinstance(finding, dict):
|
|
||||||
return False
|
|
||||||
|
|
||||||
return all(
|
|
||||||
isinstance(finding.get(key), str) and finding.get(key) for key in FINDING_KEYS
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _installed_revision(
|
|
||||||
dist: importlib.metadata.Distribution, origin: object
|
|
||||||
) -> Optional[str]:
|
|
||||||
"""Return what a plugin's installed revision is right now.
|
|
||||||
|
|
||||||
This is the value a finding recorded as installed when it was made, so a
|
|
||||||
finding can be compared against the current state of the installation
|
|
||||||
without looking anything up remotely.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
if origin != "git":
|
|
||||||
return dist.version
|
|
||||||
|
|
||||||
vcs_info = (distribution_direct_url(dist) or {}).get("vcs_info")
|
|
||||||
if not isinstance(vcs_info, dict):
|
|
||||||
return None
|
|
||||||
|
|
||||||
commit = vcs_info.get("commit_id") or ""
|
|
||||||
return commit[:SHORT_COMMIT_LENGTH] or None
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("Failed to read the installed revision of a plugin: %s", e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _batch_mode_ssh_command(ssh_command: str) -> str:
|
|
||||||
"""Return an ssh command line which cannot stop to ask a question.
|
|
||||||
|
|
||||||
ssh keeps the first value it is given for a keyword, so the batch mode
|
|
||||||
options are inserted right after the ssh program, ahead of whatever the
|
|
||||||
analyst configured. Their remaining options, such as which key to use,
|
|
||||||
still apply.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
arguments = shlex.split(ssh_command.strip())
|
|
||||||
except ValueError:
|
|
||||||
arguments = []
|
|
||||||
|
|
||||||
if not arguments:
|
|
||||||
arguments = ["ssh"]
|
|
||||||
|
|
||||||
return shlex.join([arguments[0], *_SSH_BATCH_OPTIONS, *arguments[1:]])
|
|
||||||
|
|
||||||
|
|
||||||
def _revision_pins_commit(revision: str, commit: str) -> bool:
|
|
||||||
"""Check whether a requested revision pins the installed commit."""
|
|
||||||
candidate = revision.lower()
|
|
||||||
if not _COMMIT_PATTERN.match(candidate):
|
|
||||||
return False
|
|
||||||
|
|
||||||
return commit.lower().startswith(candidate)
|
|
||||||
|
|
||||||
|
|
||||||
class PluginUpdates:
|
|
||||||
"""Check for updates to the installed MVT plugin packages.
|
|
||||||
|
|
||||||
MVT never installs or upgrades a plugin package itself. It only reports
|
|
||||||
the command which upgrades a plugin, leaving the analyst to decide when to
|
|
||||||
run it.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@property
|
|
||||||
def latest_check_path(self) -> str:
|
|
||||||
return os.path.join(MVT_DATA_FOLDER, "latest_plugins_check")
|
|
||||||
|
|
||||||
@property
|
|
||||||
def findings_path(self) -> str:
|
|
||||||
return os.path.join(MVT_DATA_FOLDER, "plugin_updates.json")
|
|
||||||
|
|
||||||
def _create_data_folder(self) -> None:
|
|
||||||
if not os.path.exists(MVT_DATA_FOLDER):
|
|
||||||
os.makedirs(MVT_DATA_FOLDER)
|
|
||||||
|
|
||||||
def get_latest_check(self) -> int:
|
|
||||||
if not os.path.exists(self.latest_check_path):
|
|
||||||
return 0
|
|
||||||
|
|
||||||
# A corrupt or truncated timestamp only means the next check happens
|
|
||||||
# sooner. It must never stop MVT from running.
|
|
||||||
try:
|
|
||||||
with open(self.latest_check_path, "r", encoding="utf-8") as handle:
|
|
||||||
data = handle.read().strip()
|
|
||||||
if data:
|
|
||||||
return int(data)
|
|
||||||
except (OSError, ValueError) as e:
|
|
||||||
log.debug("Failed to read the time of the latest plugin check: %s", e)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def set_latest_check(self) -> None:
|
|
||||||
self._create_data_folder()
|
|
||||||
timestamp = int(datetime.now().timestamp())
|
|
||||||
with open(self.latest_check_path, "w", encoding="utf-8") as handle:
|
|
||||||
handle.write(str(timestamp))
|
|
||||||
|
|
||||||
def get_findings(self) -> list[dict]:
|
|
||||||
"""
|
|
||||||
Return the findings of the latest check, without checking again.
|
|
||||||
Returns an empty list if no check was ever performed.
|
|
||||||
"""
|
|
||||||
if not os.path.exists(self.findings_path):
|
|
||||||
return []
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(self.findings_path, "r", encoding="utf-8") as handle:
|
|
||||||
findings = json.load(handle)
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("Failed to read the cached plugin updates: %s", e)
|
|
||||||
return []
|
|
||||||
|
|
||||||
if not isinstance(findings, list):
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Anything which does not look like a finding is dropped rather than
|
|
||||||
# trusted: the cache is only a convenience.
|
|
||||||
return [finding for finding in findings if _is_usable_finding(finding)]
|
|
||||||
|
|
||||||
def current_findings(
|
|
||||||
self, distributions: Optional[list[importlib.metadata.Distribution]] = None
|
|
||||||
) -> list[dict]:
|
|
||||||
"""
|
|
||||||
Return the cached findings which still apply to what is installed.
|
|
||||||
Findings about a plugin which was upgraded or removed since the latest
|
|
||||||
check are dropped, so an update is never reported twice.
|
|
||||||
"""
|
|
||||||
if distributions is None:
|
|
||||||
distributions = installed_plugin_distributions()
|
|
||||||
|
|
||||||
installed = {}
|
|
||||||
for dist in distributions:
|
|
||||||
try:
|
|
||||||
installed[dist.name] = dist
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
|
|
||||||
current = []
|
|
||||||
for finding in self.get_findings():
|
|
||||||
plugin = installed.get(finding["name"])
|
|
||||||
if plugin is None:
|
|
||||||
continue
|
|
||||||
if (
|
|
||||||
_installed_revision(plugin, finding.get("origin"))
|
|
||||||
!= finding["installed"]
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
current.append(finding)
|
|
||||||
|
|
||||||
return current
|
|
||||||
|
|
||||||
def set_findings(self, findings: list[dict]) -> None:
|
|
||||||
self._create_data_folder()
|
|
||||||
with open(self.findings_path, "w", encoding="utf-8") as handle:
|
|
||||||
json.dump(findings, handle)
|
|
||||||
|
|
||||||
def should_check(self) -> Tuple[bool, int]:
|
|
||||||
"""
|
|
||||||
Compare time of the latest plugins check with current time.
|
|
||||||
Returns bool and number of hours since the last check.
|
|
||||||
"""
|
|
||||||
now = datetime.now()
|
|
||||||
latest_check_ts = self.get_latest_check()
|
|
||||||
latest_check_dt = datetime.fromtimestamp(latest_check_ts)
|
|
||||||
|
|
||||||
diff = now - latest_check_dt
|
|
||||||
diff_hours = divmod(diff.total_seconds(), 3600)[0]
|
|
||||||
|
|
||||||
if diff_hours >= PLUGINS_CHECK_FREQUENCY:
|
|
||||||
return True, 0
|
|
||||||
|
|
||||||
return False, int(PLUGINS_CHECK_FREQUENCY - diff_hours)
|
|
||||||
|
|
||||||
def _check_index_plugin(self, name: str, installed: str) -> Optional[dict]:
|
|
||||||
"""Check a plugin installed from a package index for a newer release."""
|
|
||||||
url = f"https://pypi.org/pypi/{name}/json"
|
|
||||||
try:
|
|
||||||
res = requests.get(url, timeout=settings.NETWORK_TIMEOUT)
|
|
||||||
except requests.exceptions.RequestException as e:
|
|
||||||
log.debug("Failed to check for updates to plugin %s: %s", name, e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Plugins which were never published to a public index are expected,
|
|
||||||
# and there is nothing to compare their version against.
|
|
||||||
if res.status_code == 404:
|
|
||||||
return None
|
|
||||||
|
|
||||||
if res.status_code != 200:
|
|
||||||
log.debug(
|
|
||||||
"Failed to check for updates to plugin %s (error %d)",
|
|
||||||
name,
|
|
||||||
res.status_code,
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
|
||||||
latest = res.json().get("info", {}).get("version", "")
|
|
||||||
if not latest or version.parse(latest) <= version.parse(installed):
|
|
||||||
return None
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("Failed to compare the versions of plugin %s: %s", name, e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
return {
|
|
||||||
"name": name,
|
|
||||||
"installed": installed,
|
|
||||||
"latest": latest,
|
|
||||||
"origin": "pypi",
|
|
||||||
# The name comes from package metadata, so the command MVT
|
|
||||||
# suggests is quoted rather than assumed to be shell-safe.
|
|
||||||
"upgrade_command": f"pip install -U {shlex.quote(name)}",
|
|
||||||
}
|
|
||||||
|
|
||||||
def _git_ls_remote(self, url: str, revision: str) -> list[Tuple[str, str]]:
|
|
||||||
"""Return the remote references matching a revision, if git allows it."""
|
|
||||||
# Neither value is trusted: they are read from the metadata of an
|
|
||||||
# installed package and must not turn into git options.
|
|
||||||
if url.startswith("-") or revision.startswith("-"):
|
|
||||||
log.debug("Skipping the update check for the invalid repository %s", url)
|
|
||||||
return []
|
|
||||||
|
|
||||||
environment = dict(os.environ)
|
|
||||||
# Never prompt the analyst for repository credentials. git handles its
|
|
||||||
# own prompts, while ssh reads the terminal directly and only batch
|
|
||||||
# mode makes it fail instead of asking.
|
|
||||||
environment["GIT_TERMINAL_PROMPT"] = "0"
|
|
||||||
environment["GIT_SSH_COMMAND"] = _batch_mode_ssh_command(
|
|
||||||
environment.get("GIT_SSH_COMMAND", "")
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
process = subprocess.run(
|
|
||||||
["git", "ls-remote", url, revision],
|
|
||||||
capture_output=True,
|
|
||||||
stdin=subprocess.DEVNULL,
|
|
||||||
text=True,
|
|
||||||
env=environment,
|
|
||||||
timeout=settings.NETWORK_TIMEOUT,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
except FileNotFoundError:
|
|
||||||
log.debug("Could not find git, skipping the update check for %s", url)
|
|
||||||
return []
|
|
||||||
except (subprocess.SubprocessError, OSError) as e:
|
|
||||||
log.debug("Failed to query the repository %s: %s", url, e)
|
|
||||||
return []
|
|
||||||
|
|
||||||
if process.returncode != 0:
|
|
||||||
log.debug(
|
|
||||||
"Failed to query the repository %s (error %d): %s",
|
|
||||||
url,
|
|
||||||
process.returncode,
|
|
||||||
(process.stderr or "").strip(),
|
|
||||||
)
|
|
||||||
return []
|
|
||||||
|
|
||||||
references = []
|
|
||||||
for line in (process.stdout or "").splitlines():
|
|
||||||
commit, _, reference = line.partition("\t")
|
|
||||||
if commit.strip() and reference.strip():
|
|
||||||
references.append((commit.strip(), reference.strip()))
|
|
||||||
|
|
||||||
return references
|
|
||||||
|
|
||||||
def _check_repository_plugin(
|
|
||||||
self, name: str, direct_url: dict, vcs_info: dict
|
|
||||||
) -> Optional[dict]:
|
|
||||||
"""Check a plugin installed from a repository for a newer commit."""
|
|
||||||
url = direct_url.get("url") or ""
|
|
||||||
installed = vcs_info.get("commit_id") or ""
|
|
||||||
revision = vcs_info.get("requested_revision") or ""
|
|
||||||
if not url or not installed:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# A plugin installed from a commit is pinned and never goes out of
|
|
||||||
# date, no matter what the branch it came from does next.
|
|
||||||
if revision and _revision_pins_commit(revision, installed):
|
|
||||||
return None
|
|
||||||
|
|
||||||
latest = ""
|
|
||||||
wanted_reference = f"refs/heads/{revision}" if revision else "HEAD"
|
|
||||||
for commit, reference in self._git_ls_remote(url, revision or "HEAD"):
|
|
||||||
# Tags are pinned installs too.
|
|
||||||
if reference.startswith("refs/tags/"):
|
|
||||||
return None
|
|
||||||
if reference == wanted_reference:
|
|
||||||
latest = commit
|
|
||||||
|
|
||||||
if not latest or latest == installed:
|
|
||||||
return None
|
|
||||||
|
|
||||||
requirement = f"{name} @ git+{url}"
|
|
||||||
if revision:
|
|
||||||
requirement += f"@{revision}"
|
|
||||||
|
|
||||||
return {
|
|
||||||
"name": name,
|
|
||||||
"installed": installed[:SHORT_COMMIT_LENGTH],
|
|
||||||
"latest": latest[:SHORT_COMMIT_LENGTH],
|
|
||||||
"origin": "git",
|
|
||||||
# A repository URL and a branch name can both hold characters a
|
|
||||||
# shell would act on, so the requirement is quoted for the shell
|
|
||||||
# the analyst is going to paste the command into.
|
|
||||||
"upgrade_command": f"pip install -U {shlex.quote(requirement)}",
|
|
||||||
}
|
|
||||||
|
|
||||||
def _check_distribution(
|
|
||||||
self, dist: importlib.metadata.Distribution
|
|
||||||
) -> Optional[dict]:
|
|
||||||
try:
|
|
||||||
name = dist.name
|
|
||||||
installed = dist.version
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("Failed to read the metadata of an installed plugin: %s", e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
direct_url = distribution_direct_url(dist)
|
|
||||||
if direct_url is None:
|
|
||||||
return self._check_index_plugin(name, installed)
|
|
||||||
|
|
||||||
vcs_info = direct_url.get("vcs_info")
|
|
||||||
if isinstance(vcs_info, dict):
|
|
||||||
return self._check_repository_plugin(name, direct_url, vcs_info)
|
|
||||||
|
|
||||||
# Plugins installed from a local folder, including editable installs,
|
|
||||||
# are maintained by the analyst and have nothing to check against.
|
|
||||||
return None
|
|
||||||
|
|
||||||
def check(self) -> list[dict]:
|
|
||||||
"""
|
|
||||||
Check every installed plugin package for an available update.
|
|
||||||
Returns one entry per plugin which can be upgraded.
|
|
||||||
"""
|
|
||||||
findings = []
|
|
||||||
for dist in installed_plugin_distributions():
|
|
||||||
finding = self._check_distribution(dist)
|
|
||||||
if finding:
|
|
||||||
findings.append(finding)
|
|
||||||
|
|
||||||
self.set_findings(findings)
|
|
||||||
self.set_latest_check()
|
|
||||||
|
|
||||||
return findings
|
|
||||||
|
|
||||||
|
|
||||||
class IndicatorsUpdates:
|
class IndicatorsUpdates:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}"
|
self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}"
|
||||||
@@ -595,7 +180,9 @@ class IndicatorsUpdates:
|
|||||||
def _get_remote_file_latest_commit(
|
def _get_remote_file_latest_commit(
|
||||||
self, owner: str, repo: str, branch: str, path: str
|
self, owner: str, repo: str, branch: str, path: str
|
||||||
) -> int:
|
) -> int:
|
||||||
file_commit_url = f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}"
|
file_commit_url = (
|
||||||
|
f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}"
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
res = requests.get(file_commit_url, timeout=5)
|
res = requests.get(file_commit_url, timeout=5)
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
|
|||||||
+6
-15
@@ -14,6 +14,7 @@ from dataclasses import asdict, is_dataclass
|
|||||||
from typing import Any, Iterator, Union
|
from typing import Any, Iterator, Union
|
||||||
|
|
||||||
from .log import MVTLogHandler
|
from .log import MVTLogHandler
|
||||||
|
from mvt.common.config import settings
|
||||||
|
|
||||||
|
|
||||||
class CustomJSONEncoder(json.JSONEncoder):
|
class CustomJSONEncoder(json.JSONEncoder):
|
||||||
@@ -255,26 +256,16 @@ def init_logging(verbose: bool = False):
|
|||||||
|
|
||||||
|
|
||||||
def set_verbose_logging(verbose: bool = False):
|
def set_verbose_logging(verbose: bool = False):
|
||||||
"""Raise or lower the verbosity of MVT's console output.
|
|
||||||
|
|
||||||
Only MVT's own console handler is adjusted, wherever it sits in the list.
|
|
||||||
The file handler a command attaches to its output folder keeps recording
|
|
||||||
everything, so the command.log of a run does not depend on how the run was
|
|
||||||
invoked, and a handler attached to the "mvt" logger by anything else is
|
|
||||||
left alone.
|
|
||||||
"""
|
|
||||||
log = logging.getLogger("mvt")
|
log = logging.getLogger("mvt")
|
||||||
for handler in log.handlers:
|
handler = log.handlers[0]
|
||||||
if isinstance(handler, MVTLogHandler):
|
if verbose:
|
||||||
handler.setLevel(logging.DEBUG if verbose else logging.INFO)
|
handler.setLevel(logging.DEBUG)
|
||||||
|
else:
|
||||||
|
handler.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
def exec_or_profile(module, globals, locals):
|
def exec_or_profile(module, globals, locals):
|
||||||
"""Hook for profiling MVT modules"""
|
"""Hook for profiling MVT modules"""
|
||||||
# Imported here so that the CLI modules, which import this one at start-up,
|
|
||||||
# do not load the settings (and pydantic) before a command runs.
|
|
||||||
from .config import settings
|
|
||||||
|
|
||||||
if settings.PROFILE:
|
if settings.PROFILE:
|
||||||
cProfile.runctx(module, globals, locals)
|
cProfile.runctx(module, globals, locals)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -3,6 +3,4 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from importlib.metadata import version
|
MVT_VERSION = "2026.7.29"
|
||||||
|
|
||||||
MVT_VERSION = version("mvt")
|
|
||||||
+68
-62
@@ -14,7 +14,16 @@ from mvt.common.cli_plugins import (
|
|||||||
load_cli_commands_option,
|
load_cli_commands_option,
|
||||||
register_cli_plugins,
|
register_cli_plugins,
|
||||||
)
|
)
|
||||||
|
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||||
|
from mvt.common.completion import (
|
||||||
|
SUPPORTED_SHELLS,
|
||||||
|
completion_instructions,
|
||||||
|
generate_completion_script,
|
||||||
|
install_completion_script,
|
||||||
|
)
|
||||||
|
from mvt.common.logo import logo
|
||||||
from mvt.common.options import MutuallyExclusiveOption
|
from mvt.common.options import MutuallyExclusiveOption
|
||||||
|
from mvt.common.updates import IndicatorsUpdates
|
||||||
from mvt.common.utils import (
|
from mvt.common.utils import (
|
||||||
generate_hashes_from_path,
|
generate_hashes_from_path,
|
||||||
init_logging,
|
init_logging,
|
||||||
@@ -24,7 +33,6 @@ from mvt.common.help import (
|
|||||||
HELP_MSG_VERSION,
|
HELP_MSG_VERSION,
|
||||||
HELP_MSG_DECRYPT_BACKUP,
|
HELP_MSG_DECRYPT_BACKUP,
|
||||||
HELP_MSG_BACKUP_DESTINATION,
|
HELP_MSG_BACKUP_DESTINATION,
|
||||||
HELP_MSG_DECRYPT_JOBS,
|
|
||||||
HELP_MSG_IOS_BACKUP_PASSWORD,
|
HELP_MSG_IOS_BACKUP_PASSWORD,
|
||||||
HELP_MSG_BACKUP_KEYFILE,
|
HELP_MSG_BACKUP_KEYFILE,
|
||||||
HELP_MSG_HASHES,
|
HELP_MSG_HASHES,
|
||||||
@@ -36,7 +44,6 @@ from mvt.common.help import (
|
|||||||
HELP_MSG_LOAD_MODULE,
|
HELP_MSG_LOAD_MODULE,
|
||||||
HELP_MSG_MODULE,
|
HELP_MSG_MODULE,
|
||||||
HELP_MSG_VERBOSE,
|
HELP_MSG_VERBOSE,
|
||||||
HELP_MSG_VERBOSE_COMMAND,
|
|
||||||
HELP_MSG_CHECK_FS,
|
HELP_MSG_CHECK_FS,
|
||||||
HELP_MSG_CHECK_IOCS,
|
HELP_MSG_CHECK_IOCS,
|
||||||
HELP_MSG_STIX2,
|
HELP_MSG_STIX2,
|
||||||
@@ -44,17 +51,17 @@ from mvt.common.help import (
|
|||||||
HELP_MSG_CHECK_SYSDIAGNOSE,
|
HELP_MSG_CHECK_SYSDIAGNOSE,
|
||||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
HELP_MSG_DISABLE_UPDATE_CHECK,
|
||||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||||
|
HELP_MSG_COMPLETION,
|
||||||
)
|
)
|
||||||
|
from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
|
||||||
from mvt.common.password import prompt_password
|
from mvt.common.password import prompt_password
|
||||||
from .decrypt_config import (
|
from .cmd_check_backup import CmdIOSCheckBackup
|
||||||
DEFAULT_DECRYPT_WORKERS,
|
from .cmd_check_fs import CmdIOSCheckFS
|
||||||
MAX_DECRYPT_WORKERS,
|
from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
|
||||||
)
|
from .decrypt import DecryptBackup
|
||||||
|
from .modules.backup import BACKUP_MODULES
|
||||||
# The commands import what they run only when they are invoked. This module is
|
from .modules.fs import FS_MODULES
|
||||||
# imported at every start of mvt-ios, including by shell completion on every
|
from .modules.mixed import MIXED_MODULES
|
||||||
# keystroke, so importing it must do no more than build the command tree: the
|
|
||||||
# forensic modules, the backup decryption and the update checks stay out of it.
|
|
||||||
|
|
||||||
init_logging()
|
init_logging()
|
||||||
log = logging.getLogger("mvt")
|
log = logging.getLogger("mvt")
|
||||||
@@ -74,14 +81,7 @@ def _get_disable_flags(ctx):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_verbose(ctx):
|
|
||||||
"""Return whether --verbose was passed to the CLI itself."""
|
|
||||||
return bool(ctx.obj and ctx.obj.get("verbose", False))
|
|
||||||
|
|
||||||
|
|
||||||
def _load_custom_modules(load_module):
|
def _load_custom_modules(load_module):
|
||||||
from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return load_custom_modules(load_module)
|
return load_custom_modules(load_module)
|
||||||
except CustomModuleLoadError as exc:
|
except CustomModuleLoadError as exc:
|
||||||
@@ -101,31 +101,60 @@ def _load_custom_modules(load_module):
|
|||||||
is_flag=True,
|
is_flag=True,
|
||||||
help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
help=HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||||
)
|
)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def cli(ctx, disable_update_check, disable_indicator_update_check, verbose):
|
def cli(ctx, disable_update_check, disable_indicator_update_check):
|
||||||
ctx.ensure_object(dict)
|
ctx.ensure_object(dict)
|
||||||
ctx.obj["disable_version_check"] = disable_update_check
|
ctx.obj["disable_version_check"] = disable_update_check
|
||||||
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
ctx.obj["disable_indicator_check"] = disable_indicator_update_check
|
||||||
ctx.obj["verbose"] = verbose
|
if ctx.invoked_subcommand != "completion":
|
||||||
set_verbose_logging(verbose)
|
logo(
|
||||||
|
disable_version_check=disable_update_check,
|
||||||
from mvt.common.logo import logo
|
disable_indicator_check=disable_indicator_update_check,
|
||||||
|
)
|
||||||
logo(
|
|
||||||
disable_version_check=disable_update_check,
|
|
||||||
disable_indicator_check=disable_indicator_update_check,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# Command: version
|
# Command: version
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@cli.command("version", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_VERSION)
|
@cli.command("version", help=HELP_MSG_VERSION)
|
||||||
def version():
|
def version():
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Command: completion
|
||||||
|
# ==============================================================================
|
||||||
|
@cli.command("completion", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_COMPLETION)
|
||||||
|
@click.argument("shell", required=False, type=click.Choice(SUPPORTED_SHELLS))
|
||||||
|
@click.option(
|
||||||
|
"--install",
|
||||||
|
is_flag=True,
|
||||||
|
help="Write completion files and update shell configuration.",
|
||||||
|
)
|
||||||
|
@click.pass_context
|
||||||
|
def completion(ctx, shell, install):
|
||||||
|
program_name = "mvt-ios"
|
||||||
|
|
||||||
|
if shell is None:
|
||||||
|
if install:
|
||||||
|
raise click.UsageError("A shell is required when using --install.")
|
||||||
|
click.echo(completion_instructions(program_name))
|
||||||
|
return
|
||||||
|
|
||||||
|
root_cli = ctx.find_root().command
|
||||||
|
|
||||||
|
if install:
|
||||||
|
script_path = install_completion_script(root_cli, program_name, shell)
|
||||||
|
click.echo(f"Installed {shell} completion to {script_path}")
|
||||||
|
if shell in ("bash", "zsh"):
|
||||||
|
click.echo(f"Updated ~/.{shell}rc")
|
||||||
|
else:
|
||||||
|
click.echo("Fish loads completion files automatically.")
|
||||||
|
return
|
||||||
|
|
||||||
|
click.echo(generate_completion_script(root_cli, program_name, shell))
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# Command: decrypt-backup
|
# Command: decrypt-backup
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@@ -133,13 +162,6 @@ def version():
|
|||||||
"decrypt-backup", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_DECRYPT_BACKUP
|
"decrypt-backup", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_DECRYPT_BACKUP
|
||||||
)
|
)
|
||||||
@click.option("--destination", "-d", required=True, help=HELP_MSG_BACKUP_DESTINATION)
|
@click.option("--destination", "-d", required=True, help=HELP_MSG_BACKUP_DESTINATION)
|
||||||
@click.option(
|
|
||||||
"--jobs",
|
|
||||||
type=click.IntRange(1, MAX_DECRYPT_WORKERS),
|
|
||||||
default=DEFAULT_DECRYPT_WORKERS,
|
|
||||||
show_default=True,
|
|
||||||
help=HELP_MSG_DECRYPT_JOBS,
|
|
||||||
)
|
|
||||||
@click.option(
|
@click.option(
|
||||||
"--password",
|
"--password",
|
||||||
"-p",
|
"-p",
|
||||||
@@ -158,10 +180,8 @@ def version():
|
|||||||
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
||||||
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def decrypt_backup(ctx, destination, jobs, password, key_file, hashes, backup_path):
|
def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
|
||||||
from .decrypt import DecryptBackup
|
backup = DecryptBackup(backup_path, destination)
|
||||||
|
|
||||||
backup = DecryptBackup(backup_path, destination, max_workers=jobs)
|
|
||||||
|
|
||||||
if key_file:
|
if key_file:
|
||||||
if MVT_IOS_BACKUP_PASSWORD in os.environ:
|
if MVT_IOS_BACKUP_PASSWORD in os.environ:
|
||||||
@@ -224,8 +244,6 @@ def decrypt_backup(ctx, destination, jobs, password, key_file, hashes, backup_pa
|
|||||||
)
|
)
|
||||||
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
||||||
def extract_key(password, key_file, backup_path):
|
def extract_key(password, key_file, backup_path):
|
||||||
from .decrypt import DecryptBackup
|
|
||||||
|
|
||||||
backup = DecryptBackup(backup_path)
|
backup = DecryptBackup(backup_path)
|
||||||
|
|
||||||
if password:
|
if password:
|
||||||
@@ -278,7 +296,7 @@ def extract_key(password, key_file, backup_path):
|
|||||||
help=HELP_MSG_LOAD_MODULE,
|
help=HELP_MSG_LOAD_MODULE,
|
||||||
)
|
)
|
||||||
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
@click.argument("BACKUP_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_backup(
|
def check_backup(
|
||||||
@@ -293,9 +311,7 @@ def check_backup(
|
|||||||
verbose,
|
verbose,
|
||||||
backup_path,
|
backup_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_backup import CmdIOSCheckBackup
|
set_verbose_logging(verbose)
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
module_options = {"fast_mode": fast}
|
module_options = {"fast_mode": fast}
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
|
|
||||||
@@ -349,7 +365,7 @@ def check_backup(
|
|||||||
help=HELP_MSG_LOAD_MODULE,
|
help=HELP_MSG_LOAD_MODULE,
|
||||||
)
|
)
|
||||||
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("DUMP_PATH", type=click.Path(exists=True))
|
@click.argument("DUMP_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_fs(
|
def check_fs(
|
||||||
@@ -364,9 +380,7 @@ def check_fs(
|
|||||||
verbose,
|
verbose,
|
||||||
dump_path,
|
dump_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_fs import CmdIOSCheckFS
|
set_verbose_logging(verbose)
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
module_options = {"fast_mode": fast}
|
module_options = {"fast_mode": fast}
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
|
|
||||||
@@ -420,7 +434,7 @@ def check_fs(
|
|||||||
help=HELP_MSG_LOAD_MODULE,
|
help=HELP_MSG_LOAD_MODULE,
|
||||||
)
|
)
|
||||||
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
||||||
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
@click.argument("SYSDIAGNOSE_PATH", type=click.Path(exists=True))
|
@click.argument("SYSDIAGNOSE_PATH", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_sysdiagnose(
|
def check_sysdiagnose(
|
||||||
@@ -434,9 +448,7 @@ def check_sysdiagnose(
|
|||||||
verbose,
|
verbose,
|
||||||
sysdiagnose_path,
|
sysdiagnose_path,
|
||||||
):
|
):
|
||||||
from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
|
set_verbose_logging(verbose)
|
||||||
|
|
||||||
set_verbose_logging(verbose or _get_verbose(ctx))
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
cmd = CmdIOSCheckSysdiagnose(
|
cmd = CmdIOSCheckSysdiagnose(
|
||||||
target_path=sysdiagnose_path,
|
target_path=sysdiagnose_path,
|
||||||
@@ -490,10 +502,6 @@ def check_sysdiagnose(
|
|||||||
@click.argument("FOLDER", type=click.Path(exists=True))
|
@click.argument("FOLDER", type=click.Path(exists=True))
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
||||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
|
||||||
|
|
||||||
from .command_modules import IOS_CHECK_IOCS_MODULES
|
|
||||||
|
|
||||||
custom_modules = _load_custom_modules(load_module)
|
custom_modules = _load_custom_modules(load_module)
|
||||||
cmd = CmdCheckIOCS(
|
cmd = CmdCheckIOCS(
|
||||||
target_path=folder,
|
target_path=folder,
|
||||||
@@ -504,7 +512,7 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
|||||||
custom_modules=custom_modules,
|
custom_modules=custom_modules,
|
||||||
platform="ios",
|
platform="ios",
|
||||||
)
|
)
|
||||||
cmd.modules = IOS_CHECK_IOCS_MODULES
|
cmd.modules = BACKUP_MODULES + FS_MODULES + MIXED_MODULES
|
||||||
|
|
||||||
if list_modules:
|
if list_modules:
|
||||||
cmd.list_modules()
|
cmd.list_modules()
|
||||||
@@ -520,8 +528,6 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
|||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
|
@cli.command("download-iocs", context_settings=CONTEXT_SETTINGS, help=HELP_MSG_STIX2)
|
||||||
def download_iocs():
|
def download_iocs():
|
||||||
from mvt.common.updates import IndicatorsUpdates
|
|
||||||
|
|
||||||
ioc_updates = IndicatorsUpdates()
|
ioc_updates = IndicatorsUpdates()
|
||||||
ioc_updates.update()
|
ioc_updates.update()
|
||||||
|
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
"""Module lists an mvt-ios command composes from more than one family.
|
|
||||||
|
|
||||||
Commands whose modules are one family read that family directly. check-iocs
|
|
||||||
re-checks stored results, so it has to know every module that could have
|
|
||||||
written one, and both the CLI and any other code needing that answer share
|
|
||||||
the list from here rather than each concatenating their own.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .modules.backup import BACKUP_MODULES
|
|
||||||
from .modules.fs import FS_MODULES
|
|
||||||
from .modules.mixed import MIXED_MODULES
|
|
||||||
|
|
||||||
IOS_CHECK_IOCS_MODULES: list[type[MVTModule]] = (
|
|
||||||
BACKUP_MODULES + FS_MODULES + MIXED_MODULES
|
|
||||||
)
|
|
||||||
+74
-292
@@ -6,179 +6,18 @@
|
|||||||
import binascii
|
import binascii
|
||||||
import glob
|
import glob
|
||||||
import logging
|
import logging
|
||||||
|
import multiprocessing
|
||||||
import os
|
import os
|
||||||
import os.path
|
import os.path
|
||||||
import plistlib
|
|
||||||
import shutil
|
import shutil
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import tempfile
|
|
||||||
from concurrent.futures import (
|
|
||||||
ALL_COMPLETED,
|
|
||||||
FIRST_COMPLETED,
|
|
||||||
Future,
|
|
||||||
ThreadPoolExecutor,
|
|
||||||
wait,
|
|
||||||
)
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from iphone_backup_decrypt import EncryptedBackup
|
from iOSbackup import iOSbackup
|
||||||
from iphone_backup_decrypt import google_iphone_dataprotection
|
|
||||||
from iphone_backup_decrypt.utils import FilePlist
|
|
||||||
|
|
||||||
from .decrypt_config import DEFAULT_DECRYPT_WORKERS, MAX_DECRYPT_WORKERS
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Import pbkdf2_hmac from the same source iphone_backup_decrypt uses internally,
|
|
||||||
# so our key derivation is consistent with theirs.
|
|
||||||
try:
|
|
||||||
from fastpbkdf2 import pbkdf2_hmac
|
|
||||||
except ImportError:
|
|
||||||
import Crypto.Hash.SHA1
|
|
||||||
import Crypto.Hash.SHA256
|
|
||||||
import Crypto.Protocol.KDF
|
|
||||||
|
|
||||||
_HASH_FNS = {"sha1": Crypto.Hash.SHA1, "sha256": Crypto.Hash.SHA256}
|
|
||||||
|
|
||||||
def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None):
|
|
||||||
return Crypto.Protocol.KDF.PBKDF2(
|
|
||||||
password, salt, dklen, iterations, hmac_hash_module=_HASH_FNS[hash_name]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MVTEncryptedBackup(EncryptedBackup):
|
|
||||||
"""Extends EncryptedBackup with derived key export/import.
|
|
||||||
|
|
||||||
NOTE: This subclass relies on internal APIs of iphone_backup_decrypt
|
|
||||||
(specifically _read_and_unlock_keybag, _keybag, and the Keybag class
|
|
||||||
internals). Pinned to iphone_backup_decrypt==0.9.0.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, *, backup_directory, passphrase=None, derived_key=None):
|
|
||||||
if passphrase:
|
|
||||||
super().__init__(backup_directory=backup_directory, passphrase=passphrase)
|
|
||||||
self._derived_key = None # Will be set after keybag unlock
|
|
||||||
elif derived_key:
|
|
||||||
self._init_without_passphrase(backup_directory, derived_key)
|
|
||||||
else:
|
|
||||||
raise ValueError("Either passphrase or derived_key must be provided")
|
|
||||||
|
|
||||||
def _init_without_passphrase(self, backup_directory, derived_key):
|
|
||||||
"""Replicate parent __init__ state without requiring a passphrase."""
|
|
||||||
self.decrypted = False
|
|
||||||
self._backup_directory = os.path.expandvars(backup_directory)
|
|
||||||
self._passphrase = None
|
|
||||||
self._manifest_plist_path = os.path.join(
|
|
||||||
self._backup_directory, "Manifest.plist"
|
|
||||||
)
|
|
||||||
self._manifest_plist = None
|
|
||||||
self._manifest_db_path = os.path.join(self._backup_directory, "Manifest.db")
|
|
||||||
self._keybag = None
|
|
||||||
self._unlocked = False
|
|
||||||
self._temporary_folder = tempfile.mkdtemp()
|
|
||||||
self._temp_decrypted_manifest_db_path = os.path.join(
|
|
||||||
self._temporary_folder, "Manifest.db"
|
|
||||||
)
|
|
||||||
self._temp_manifest_db_conn = None
|
|
||||||
self._derived_key = derived_key # 32 raw bytes
|
|
||||||
|
|
||||||
def _read_and_unlock_keybag(self):
|
|
||||||
"""Override to capture derived key on password unlock, or use
|
|
||||||
a pre-derived key to skip PBKDF2."""
|
|
||||||
if self._unlocked:
|
|
||||||
return self._unlocked
|
|
||||||
|
|
||||||
with open(self._manifest_plist_path, "rb") as infile:
|
|
||||||
self._manifest_plist = plistlib.load(infile)
|
|
||||||
self._keybag = google_iphone_dataprotection.Keybag(
|
|
||||||
self._manifest_plist["BackupKeyBag"]
|
|
||||||
)
|
|
||||||
|
|
||||||
if self._derived_key:
|
|
||||||
# Skip PBKDF2, unwrap class keys directly with pre-derived key
|
|
||||||
self._unlocked = _unlock_keybag_with_derived_key(
|
|
||||||
self._keybag, self._derived_key
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Normal path: full PBKDF2 derivation, capturing the intermediate key
|
|
||||||
self._unlocked, self._derived_key = _unlock_keybag_and_capture_key(
|
|
||||||
self._keybag, self._passphrase
|
|
||||||
)
|
|
||||||
self._passphrase = None
|
|
||||||
|
|
||||||
if not self._unlocked:
|
|
||||||
raise ValueError("Failed to decrypt keys: incorrect passphrase?")
|
|
||||||
return True
|
|
||||||
|
|
||||||
def get_decryption_key(self):
|
|
||||||
"""Return derived key as hex string (64 chars / 32 bytes)."""
|
|
||||||
if self._derived_key is None:
|
|
||||||
raise ValueError("No derived key available")
|
|
||||||
return self._derived_key.hex()
|
|
||||||
|
|
||||||
def extract_file_by_id(self, *, file_id, file_bplist, output_filename):
|
|
||||||
"""Extract one manifest entry without loading the whole file into memory."""
|
|
||||||
self._read_and_unlock_keybag()
|
|
||||||
file_plist = FilePlist(file_bplist)
|
|
||||||
|
|
||||||
if file_plist.encryption_key is None:
|
|
||||||
source_filename = os.path.join(
|
|
||||||
self._backup_directory, file_id[:2], file_id
|
|
||||||
)
|
|
||||||
shutil.copy2(source_filename, output_filename)
|
|
||||||
return
|
|
||||||
|
|
||||||
inner_key = self._keybag.unwrapKeyForClass(
|
|
||||||
file_plist.protection_class, file_plist.encryption_key
|
|
||||||
)
|
|
||||||
self._decrypt_file_to_disk(
|
|
||||||
file_id=file_id,
|
|
||||||
key=inner_key,
|
|
||||||
file_plist=file_plist,
|
|
||||||
output_filepath=output_filename,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _unlock_keybag_with_derived_key(keybag, passphrase_key):
|
|
||||||
"""Unlock keybag class keys using a pre-derived passphrase_key,
|
|
||||||
skipping the expensive PBKDF2 rounds."""
|
|
||||||
WRAP_PASSPHRASE = 2
|
|
||||||
for classkey in keybag.classKeys.values():
|
|
||||||
if b"WPKY" not in classkey:
|
|
||||||
continue
|
|
||||||
if classkey[b"WRAP"] & WRAP_PASSPHRASE:
|
|
||||||
k = google_iphone_dataprotection._AESUnwrap(
|
|
||||||
passphrase_key, classkey[b"WPKY"]
|
|
||||||
)
|
|
||||||
if not k:
|
|
||||||
return False
|
|
||||||
classkey[b"KEY"] = k
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _unlock_keybag_and_capture_key(keybag, passphrase):
|
|
||||||
"""Run full PBKDF2 key derivation and AES unwrap, returning
|
|
||||||
(success, passphrase_key) so the derived key can be exported."""
|
|
||||||
passphrase_round1 = pbkdf2_hmac(
|
|
||||||
"sha256", passphrase, keybag.attrs[b"DPSL"], keybag.attrs[b"DPIC"], 32
|
|
||||||
)
|
|
||||||
passphrase_key = pbkdf2_hmac(
|
|
||||||
"sha1", passphrase_round1, keybag.attrs[b"SALT"], keybag.attrs[b"ITER"], 32
|
|
||||||
)
|
|
||||||
WRAP_PASSPHRASE = 2
|
|
||||||
for classkey in keybag.classKeys.values():
|
|
||||||
if b"WPKY" not in classkey:
|
|
||||||
continue
|
|
||||||
if classkey[b"WRAP"] & WRAP_PASSPHRASE:
|
|
||||||
k = google_iphone_dataprotection._AESUnwrap(
|
|
||||||
passphrase_key, classkey[b"WPKY"]
|
|
||||||
)
|
|
||||||
if not k:
|
|
||||||
return False, None
|
|
||||||
classkey[b"KEY"] = k
|
|
||||||
return True, passphrase_key
|
|
||||||
|
|
||||||
|
|
||||||
class DecryptBackup:
|
class DecryptBackup:
|
||||||
"""This class provides functions to decrypt an encrypted iTunes backup
|
"""This class provides functions to decrypt an encrypted iTunes backup
|
||||||
@@ -187,68 +26,19 @@ class DecryptBackup:
|
|||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(self, backup_path: str, dest_path: Optional[str] = None) -> None:
|
||||||
self,
|
|
||||||
backup_path: str,
|
|
||||||
dest_path: Optional[str] = None,
|
|
||||||
max_workers: int = DEFAULT_DECRYPT_WORKERS,
|
|
||||||
) -> None:
|
|
||||||
"""Decrypts an encrypted iOS backup.
|
"""Decrypts an encrypted iOS backup.
|
||||||
:param backup_path: Path to the encrypted backup folder
|
:param backup_path: Path to the encrypted backup folder
|
||||||
:param dest_path: Path to the folder where to store the decrypted backup
|
:param dest_path: Path to the folder where to store the decrypted backup
|
||||||
"""
|
"""
|
||||||
self.backup_path = os.path.abspath(backup_path)
|
self.backup_path = os.path.abspath(backup_path)
|
||||||
self.dest_path = dest_path
|
self.dest_path = dest_path
|
||||||
if not 1 <= max_workers <= MAX_DECRYPT_WORKERS:
|
self._backup = None
|
||||||
raise ValueError(f"max_workers must be between 1 and {MAX_DECRYPT_WORKERS}")
|
self._decryption_key = None
|
||||||
self.max_workers = max_workers
|
|
||||||
self._backup: Optional[MVTEncryptedBackup] = None
|
|
||||||
self._decryption_key: Optional[str] = None
|
|
||||||
|
|
||||||
def can_process(self) -> bool:
|
def can_process(self) -> bool:
|
||||||
return self._backup is not None
|
return self._backup is not None
|
||||||
|
|
||||||
def _process_file(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
file_id: str,
|
|
||||||
file_bplist: bytes,
|
|
||||||
output_path: Path,
|
|
||||||
relative_path: str,
|
|
||||||
domain: str,
|
|
||||||
) -> None:
|
|
||||||
assert self._backup is not None
|
|
||||||
self._backup.extract_file_by_id(
|
|
||||||
file_id=file_id,
|
|
||||||
file_bplist=file_bplist,
|
|
||||||
output_filename=str(output_path),
|
|
||||||
)
|
|
||||||
log.info(
|
|
||||||
"Decrypted file %s [%s] to %s/%s",
|
|
||||||
relative_path,
|
|
||||||
domain,
|
|
||||||
output_path.parent,
|
|
||||||
file_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _wait_for_files(
|
|
||||||
pending: dict[Future[None], str], *, all_files: bool = False
|
|
||||||
) -> None:
|
|
||||||
if not pending:
|
|
||||||
return
|
|
||||||
|
|
||||||
done, _ = wait(
|
|
||||||
pending,
|
|
||||||
return_when=ALL_COMPLETED if all_files else FIRST_COMPLETED,
|
|
||||||
)
|
|
||||||
for future in done:
|
|
||||||
relative_path = pending.pop(future)
|
|
||||||
try:
|
|
||||||
future.result()
|
|
||||||
except Exception as exc:
|
|
||||||
log.error("Failed to decrypt file %s: %s", relative_path, exc)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def is_encrypted(backup_path: str) -> bool:
|
def is_encrypted(backup_path: str) -> bool:
|
||||||
"""Query Manifest.db file to see if it's encrypted or not.
|
"""Query Manifest.db file to see if it's encrypted or not.
|
||||||
@@ -268,6 +58,21 @@ class DecryptBackup:
|
|||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
def _process_file(
|
||||||
|
self, relative_path: str, domain: str, item, file_id: str, item_folder: str
|
||||||
|
) -> None:
|
||||||
|
assert self._backup is not None
|
||||||
|
self._backup.getFileDecryptedCopy(
|
||||||
|
manifestEntry=item, targetName=file_id, targetFolder=item_folder
|
||||||
|
)
|
||||||
|
log.info(
|
||||||
|
"Decrypted file %s [%s] to %s/%s",
|
||||||
|
relative_path,
|
||||||
|
domain,
|
||||||
|
item_folder,
|
||||||
|
file_id,
|
||||||
|
)
|
||||||
|
|
||||||
def process_backup(self) -> None:
|
def process_backup(self) -> None:
|
||||||
assert self._backup is not None
|
assert self._backup is not None
|
||||||
assert self.dest_path is not None
|
assert self.dest_path is not None
|
||||||
@@ -276,71 +81,53 @@ class DecryptBackup:
|
|||||||
os.makedirs(self.dest_path)
|
os.makedirs(self.dest_path)
|
||||||
|
|
||||||
manifest_path = os.path.join(self.dest_path, "Manifest.db")
|
manifest_path = os.path.join(self.dest_path, "Manifest.db")
|
||||||
# Extract a decrypted Manifest.db to the destination folder.
|
# We extract a decrypted Manifest.db.
|
||||||
self._backup.save_manifest_file(output_filename=manifest_path)
|
self._backup.getManifestDB()
|
||||||
|
# We store it to the destination folder.
|
||||||
|
shutil.copy(self._backup.manifestDB, manifest_path)
|
||||||
|
|
||||||
# Iterate over all files in the backup and decrypt them,
|
pool = multiprocessing.Pool(multiprocessing.cpu_count())
|
||||||
# preserving the XX/file_id directory structure that downstream
|
|
||||||
# modules expect.
|
|
||||||
backup_root = Path(self.backup_path).resolve()
|
|
||||||
dest_root = Path(self.dest_path).resolve()
|
|
||||||
pending: dict[Future[None], str] = {}
|
|
||||||
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
|
||||||
with self._backup.manifest_db_cursor() as cur:
|
|
||||||
cur.execute(
|
|
||||||
"SELECT fileID, domain, relativePath, file FROM Files WHERE flags=1"
|
|
||||||
)
|
|
||||||
for file_id, domain, relative_path, file_bplist in cur:
|
|
||||||
# This may be a partial backup. Skip files from the manifest
|
|
||||||
# which do not exist locally.
|
|
||||||
source_file_path = backup_root / file_id[:2] / file_id
|
|
||||||
if not source_file_path.resolve().is_relative_to(backup_root):
|
|
||||||
log.warning("Skipping unsafe file_id: %r", file_id)
|
|
||||||
continue
|
|
||||||
if not os.path.exists(source_file_path):
|
|
||||||
log.debug(
|
|
||||||
"Skipping file %s. File not found in encrypted "
|
|
||||||
"backup directory.",
|
|
||||||
source_file_path,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
output_path = dest_root / file_id[:2] / file_id
|
for item in self._backup.getBackupFilesList():
|
||||||
if not output_path.resolve().is_relative_to(dest_root):
|
try:
|
||||||
log.warning("Skipping unsafe file_id: %r", file_id)
|
file_id = item["backupFile"]
|
||||||
continue
|
relative_path = item["relativePath"]
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
domain = item["domain"]
|
||||||
|
|
||||||
if self.max_workers == 1:
|
# This may be a partial backup. Skip files from the manifest
|
||||||
try:
|
# which do not exist locally.
|
||||||
self._process_file(
|
source_file_path = os.path.join(self.backup_path, file_id[0:2], file_id)
|
||||||
file_id=file_id,
|
if not Path(source_file_path).resolve().is_relative_to(Path(self.backup_path).resolve()):
|
||||||
file_bplist=file_bplist,
|
log.warning("Skipping unsafe file_id: %r", file_id)
|
||||||
output_path=output_path,
|
continue
|
||||||
relative_path=relative_path,
|
if not os.path.exists(source_file_path):
|
||||||
domain=domain,
|
log.debug(
|
||||||
)
|
"Skipping file %s. File not found in encrypted backup directory.",
|
||||||
except Exception as exc:
|
source_file_path,
|
||||||
log.error(
|
|
||||||
"Failed to decrypt file %s: %s",
|
|
||||||
relative_path,
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
future = executor.submit(
|
|
||||||
self._process_file,
|
|
||||||
file_id=file_id,
|
|
||||||
file_bplist=file_bplist,
|
|
||||||
output_path=output_path,
|
|
||||||
relative_path=relative_path,
|
|
||||||
domain=domain,
|
|
||||||
)
|
)
|
||||||
pending[future] = relative_path
|
continue
|
||||||
if len(pending) >= self.max_workers:
|
|
||||||
self._wait_for_files(pending)
|
|
||||||
|
|
||||||
self._wait_for_files(pending, all_files=True)
|
item_folder = os.path.join(self.dest_path, file_id[0:2]) # type: ignore[arg-type]
|
||||||
|
if not Path(os.path.join(item_folder, file_id)).resolve().is_relative_to(Path(self.dest_path).resolve()):
|
||||||
|
log.warning("Skipping unsafe file_id: %r", file_id)
|
||||||
|
continue
|
||||||
|
if not os.path.exists(item_folder):
|
||||||
|
os.makedirs(item_folder)
|
||||||
|
|
||||||
|
# iOSBackup getFileDecryptedCopy() claims to read a "file"
|
||||||
|
# parameter but the code actually is reading the "manifest" key.
|
||||||
|
# Add manifest plist to both keys to handle this.
|
||||||
|
item["manifest"] = item["file"]
|
||||||
|
|
||||||
|
pool.apply_async(
|
||||||
|
self._process_file,
|
||||||
|
args=(relative_path, domain, item, file_id, item_folder),
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("Failed to decrypt file %s: %s", relative_path, exc)
|
||||||
|
|
||||||
|
pool.close()
|
||||||
|
pool.join()
|
||||||
|
|
||||||
# Copying over the root plist files as well.
|
# Copying over the root plist files as well.
|
||||||
for file_name in os.listdir(self.backup_path):
|
for file_name in os.listdir(self.backup_path):
|
||||||
@@ -381,23 +168,20 @@ class DecryptBackup:
|
|||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self._backup = MVTEncryptedBackup(
|
self._backup = iOSbackup(
|
||||||
backup_directory=self.backup_path,
|
udid=os.path.basename(self.backup_path),
|
||||||
passphrase=password,
|
cleartextpassword=password,
|
||||||
|
backuproot=os.path.dirname(self.backup_path),
|
||||||
)
|
)
|
||||||
# Eagerly trigger keybag unlock so wrong-password errors
|
|
||||||
# surface here rather than later during process_backup().
|
|
||||||
self._backup.test_decryption()
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._backup = None
|
|
||||||
if (
|
if (
|
||||||
isinstance(exc, ValueError)
|
isinstance(exc, KeyError)
|
||||||
and "passphrase" in str(exc).lower()
|
and len(exc.args) > 0
|
||||||
|
and exc.args[0] == b"KEY"
|
||||||
):
|
):
|
||||||
log.critical("Failed to decrypt backup. Password is probably wrong.")
|
log.critical("Failed to decrypt backup. Password is probably wrong.")
|
||||||
elif (
|
elif (
|
||||||
isinstance(exc, FileNotFoundError)
|
isinstance(exc, FileNotFoundError)
|
||||||
and hasattr(exc, "filename")
|
|
||||||
and os.path.basename(exc.filename) == "Manifest.plist"
|
and os.path.basename(exc.filename) == "Manifest.plist"
|
||||||
):
|
):
|
||||||
log.critical(
|
log.critical(
|
||||||
@@ -440,14 +224,12 @@ class DecryptBackup:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
key_bytes_raw = binascii.unhexlify(key_bytes)
|
key_bytes_raw = binascii.unhexlify(key_bytes)
|
||||||
self._backup = MVTEncryptedBackup(
|
self._backup = iOSbackup(
|
||||||
backup_directory=self.backup_path,
|
udid=os.path.basename(self.backup_path),
|
||||||
derived_key=key_bytes_raw,
|
derivedkey=key_bytes_raw,
|
||||||
|
backuproot=os.path.dirname(self.backup_path),
|
||||||
)
|
)
|
||||||
# Eagerly trigger keybag unlock so wrong-key errors surface here.
|
|
||||||
self._backup.test_decryption()
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._backup = None
|
|
||||||
log.exception(exc)
|
log.exception(exc)
|
||||||
log.critical(
|
log.critical(
|
||||||
"Failed to decrypt backup. Did you provide the correct key file?"
|
"Failed to decrypt backup. Did you provide the correct key file?"
|
||||||
@@ -458,7 +240,7 @@ class DecryptBackup:
|
|||||||
if not self._backup:
|
if not self._backup:
|
||||||
return
|
return
|
||||||
|
|
||||||
self._decryption_key = self._backup.get_decryption_key()
|
self._decryption_key = self._backup.getDecryptionKey()
|
||||||
log.info(
|
log.info(
|
||||||
'Derived decryption key for backup at path %s is: "%s"',
|
'Derived decryption key for backup at path %s is: "%s"',
|
||||||
self.backup_path,
|
self.backup_path,
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
"""Configuration shared by the iOS CLI and backup decryption implementation."""
|
|
||||||
|
|
||||||
DEFAULT_DECRYPT_WORKERS = 4
|
|
||||||
MAX_DECRYPT_WORKERS = 32
|
|
||||||
@@ -3,16 +3,9 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .backup_info import BackupInfo
|
from .backup_info import BackupInfo
|
||||||
from .configuration_profiles import ConfigurationProfiles
|
from .configuration_profiles import ConfigurationProfiles
|
||||||
from .manifest import Manifest
|
from .manifest import Manifest
|
||||||
from .profile_events import ProfileEvents
|
from .profile_events import ProfileEvents
|
||||||
|
|
||||||
BACKUP_MODULES: list[type[MVTModule]] = [
|
BACKUP_MODULES = [BackupInfo, ConfigurationProfiles, Manifest, ProfileEvents]
|
||||||
BackupInfo,
|
|
||||||
ConfigurationProfiles,
|
|
||||||
Manifest,
|
|
||||||
ProfileEvents,
|
|
||||||
]
|
|
||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .analytics import Analytics
|
from .analytics import Analytics
|
||||||
from .analytics_ios_versions import AnalyticsIOSVersions
|
from .analytics_ios_versions import AnalyticsIOSVersions
|
||||||
from .cache_files import CacheFiles
|
from .cache_files import CacheFiles
|
||||||
@@ -17,7 +15,7 @@ from .webkit_indexeddb import WebkitIndexedDB
|
|||||||
from .webkit_localstorage import WebkitLocalStorage
|
from .webkit_localstorage import WebkitLocalStorage
|
||||||
from .webkit_safariviewservice import WebkitSafariViewService
|
from .webkit_safariviewservice import WebkitSafariViewService
|
||||||
|
|
||||||
FS_MODULES: list[type[MVTModule]] = [
|
FS_MODULES = [
|
||||||
CacheFiles,
|
CacheFiles,
|
||||||
Filesystem,
|
Filesystem,
|
||||||
Netusage,
|
Netusage,
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
from .applications import Applications
|
from .applications import Applications
|
||||||
from .calendar import Calendar
|
from .calendar import Calendar
|
||||||
from .calls import Calls
|
from .calls import Calls
|
||||||
@@ -30,7 +28,7 @@ from .webkit_session_resource_log import WebkitSessionResourceLog
|
|||||||
from .whatsapp import Whatsapp
|
from .whatsapp import Whatsapp
|
||||||
from .whatsapp_contacts import WhatsappContacts
|
from .whatsapp_contacts import WhatsappContacts
|
||||||
|
|
||||||
MIXED_MODULES: list[type[MVTModule]] = [
|
MIXED_MODULES = [
|
||||||
Calls,
|
Calls,
|
||||||
ChromeFavicon,
|
ChromeFavicon,
|
||||||
ChromeHistory,
|
ChromeHistory,
|
||||||
|
|||||||
@@ -1,80 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
"""Stable functions and modules which plugins can import.
|
|
||||||
|
|
||||||
Anything else in mvt can be imported too, and may change
|
|
||||||
between releases without notice.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from mvt.android.modules.androidqf.base import AndroidQFModule
|
|
||||||
from mvt.android.modules.backup.base import BackupModule as AndroidBackupModule
|
|
||||||
from mvt.android.modules.bugreport.base import BugReportModule
|
|
||||||
from mvt.common.alerts import Alert, AlertLevel
|
|
||||||
from mvt.common.command import Command
|
|
||||||
from mvt.common.config import settings
|
|
||||||
from mvt.common.module import DatabaseCorruptedError, DatabaseNotFoundError, MVTModule
|
|
||||||
from mvt.common.module_loader import get_plugin_logger
|
|
||||||
from mvt.common.module_types import (
|
|
||||||
ModuleAtomicResult,
|
|
||||||
ModuleResults,
|
|
||||||
ModuleSerializedResult,
|
|
||||||
)
|
|
||||||
from mvt.common.plugin_config import (
|
|
||||||
MVTPluginSettings,
|
|
||||||
PluginConfigLoadError,
|
|
||||||
plugin_config_path,
|
|
||||||
plugin_data_folder,
|
|
||||||
plugin_env_prefix,
|
|
||||||
)
|
|
||||||
from mvt.common.utils import (
|
|
||||||
convert_chrometime_to_datetime,
|
|
||||||
convert_datetime_to_iso,
|
|
||||||
convert_mactime_to_datetime,
|
|
||||||
convert_mactime_to_iso,
|
|
||||||
convert_unix_to_iso,
|
|
||||||
convert_unix_to_utc_datetime,
|
|
||||||
)
|
|
||||||
from mvt.common.version import MVT_VERSION
|
|
||||||
from mvt.ios.modules.base import IOSExtraction
|
|
||||||
from mvt.ios.modules.sysdiagnose.base import SysdiagnoseExtraction
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
# Classes a plugin subclasses.
|
|
||||||
"MVTModule",
|
|
||||||
"IOSExtraction",
|
|
||||||
"SysdiagnoseExtraction",
|
|
||||||
"AndroidQFModule",
|
|
||||||
"AndroidBackupModule",
|
|
||||||
"BugReportModule",
|
|
||||||
"Command",
|
|
||||||
# Results and alerts.
|
|
||||||
"ModuleAtomicResult",
|
|
||||||
"ModuleResults",
|
|
||||||
"ModuleSerializedResult",
|
|
||||||
"Alert",
|
|
||||||
"AlertLevel",
|
|
||||||
# Errors a module raises.
|
|
||||||
"DatabaseNotFoundError",
|
|
||||||
"DatabaseCorruptedError",
|
|
||||||
# Settings.
|
|
||||||
"settings",
|
|
||||||
"MVTPluginSettings",
|
|
||||||
"PluginConfigLoadError",
|
|
||||||
"plugin_config_path",
|
|
||||||
"plugin_data_folder",
|
|
||||||
"plugin_env_prefix",
|
|
||||||
# Logging.
|
|
||||||
"get_plugin_logger",
|
|
||||||
# Timestamps.
|
|
||||||
"convert_chrometime_to_datetime",
|
|
||||||
"convert_datetime_to_iso",
|
|
||||||
"convert_mactime_to_datetime",
|
|
||||||
"convert_mactime_to_iso",
|
|
||||||
"convert_unix_to_iso",
|
|
||||||
"convert_unix_to_utc_datetime",
|
|
||||||
# MVT's version.
|
|
||||||
"MVT_VERSION",
|
|
||||||
]
|
|
||||||
@@ -23,7 +23,7 @@ class TestDumpsysAccessibilityArtifact:
|
|||||||
assert len(da.results) == 4
|
assert len(da.results) == 4
|
||||||
assert da.results[0]["package_name"] == "com.android.settings"
|
assert da.results[0]["package_name"] == "com.android.settings"
|
||||||
assert (
|
assert (
|
||||||
da.results[0]["component"]
|
da.results[0]["service"]
|
||||||
== "com.android.settings/com.samsung.android.settings.development.gpuwatch.GPUWatchInterceptor"
|
== "com.android.settings/com.samsung.android.settings.development.gpuwatch.GPUWatchInterceptor"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,9 +37,7 @@ class TestDumpsysAccessibilityArtifact:
|
|||||||
da.parse(data)
|
da.parse(data)
|
||||||
assert len(da.results) == 1
|
assert len(da.results) == 1
|
||||||
assert da.results[0]["package_name"] == "com.malware.accessibility"
|
assert da.results[0]["package_name"] == "com.malware.accessibility"
|
||||||
assert da.results[0]["service_name"] == "com.malware.service.malwareservice"
|
assert da.results[0]["service"] == "com.malware.service.malwareservice"
|
||||||
assert da.results[0]["enabled"] is True
|
|
||||||
assert da.results[0]["installed"] is False
|
|
||||||
|
|
||||||
def test_accessibility_service_alert(self):
|
def test_accessibility_service_alert(self):
|
||||||
da = DumpsysAccessibilityArtifact()
|
da = DumpsysAccessibilityArtifact()
|
||||||
@@ -54,22 +52,6 @@ class TestDumpsysAccessibilityArtifact:
|
|||||||
assert da.alertstore.alerts[0].level == AlertLevel.MEDIUM
|
assert da.alertstore.alerts[0].level == AlertLevel.MEDIUM
|
||||||
assert da.alertstore.alerts[0].event == da.results[0]
|
assert da.alertstore.alerts[0].event == da.results[0]
|
||||||
|
|
||||||
def test_same_component_is_kept_for_each_user(self):
|
|
||||||
da = DumpsysAccessibilityArtifact()
|
|
||||||
da.parse(
|
|
||||||
"""User state[attributes:{id=0
|
|
||||||
installed services: {
|
|
||||||
0 : com.example/.Service
|
|
||||||
}
|
|
||||||
User state[attributes:{id=10
|
|
||||||
installed services: {
|
|
||||||
0 : com.example/.Service
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
assert [result["user_id"] for result in da.results] == [0, 10]
|
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
da = DumpsysAccessibilityArtifact()
|
da = DumpsysAccessibilityArtifact()
|
||||||
file = get_artifact("android_data/dumpsys_accessibility.txt")
|
file = get_artifact("android_data/dumpsys_accessibility.txt")
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import base64
|
|
||||||
|
|
||||||
from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact
|
from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact
|
||||||
from mvt.android.modules.bugreport.dumpsys_adb_state import DumpsysADBState
|
from mvt.android.modules.bugreport.dumpsys_adb_state import DumpsysADBState
|
||||||
from mvt.common.alerts import AlertLevel
|
from mvt.common.alerts import AlertLevel
|
||||||
@@ -13,17 +11,6 @@ from ..utils import get_artifact
|
|||||||
|
|
||||||
|
|
||||||
class TestDumpsysADBArtifact:
|
class TestDumpsysADBArtifact:
|
||||||
def test_parsing_binary_xml_recovers_key(self):
|
|
||||||
public_key = base64.b64encode(bytes(range(256)))
|
|
||||||
keystore = DumpsysADBArtifact().parse_binary_xml(
|
|
||||||
b"ABX\x00binary-key=" + public_key + b" user@host\x00lastConnection"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert len(keystore) == 1
|
|
||||||
assert keystore[0]["key"] == public_key.decode()
|
|
||||||
assert keystore[0]["user"] == "user@host"
|
|
||||||
assert keystore[0]["last_connected"] is None
|
|
||||||
|
|
||||||
def test_parsing(self):
|
def test_parsing(self):
|
||||||
da_adb = DumpsysADBArtifact()
|
da_adb = DumpsysADBArtifact()
|
||||||
file = get_artifact("android_data/dumpsys_adb.txt")
|
file = get_artifact("android_data/dumpsys_adb.txt")
|
||||||
|
|||||||
@@ -25,37 +25,10 @@ class TestDumpsysAppopsArtifact:
|
|||||||
assert da.results[0]["uid"] == "0"
|
assert da.results[0]["uid"] == "0"
|
||||||
assert len(da.results[0]["permissions"]) == 1
|
assert len(da.results[0]["permissions"]) == 1
|
||||||
assert da.results[0]["permissions"][0]["name"] == "MANAGE_IPSEC_TUNNELS"
|
assert da.results[0]["permissions"][0]["name"] == "MANAGE_IPSEC_TUNNELS"
|
||||||
assert da.results[0]["permissions"][0]["mode"] == "allow"
|
assert da.results[0]["permissions"][0]["access"] == "allow"
|
||||||
assert da.results[6]["package_name"] == "com.sec.factory.camera"
|
assert da.results[6]["package_name"] == "com.sec.factory.camera"
|
||||||
assert len(da.results[6]["permissions"][1]["entries"]) == 1
|
assert len(da.results[6]["permissions"][1]["entries"]) == 1
|
||||||
assert len(da.results[11]["permissions"]) == 4
|
assert len(da.results[11]["permissions"]) == 4
|
||||||
wake_lock = next(
|
|
||||||
permission
|
|
||||||
for permission in da.results[11]["permissions"]
|
|
||||||
if permission["name"] == "WAKE_LOCK"
|
|
||||||
)
|
|
||||||
assert wake_lock["entries"][0]["duration"] == "+126ms"
|
|
||||||
|
|
||||||
def test_running_and_attribution_are_retained(self):
|
|
||||||
da = DumpsysAppopsArtifact()
|
|
||||||
da.parse(
|
|
||||||
""" Uid 0:
|
|
||||||
state=cch
|
|
||||||
Package com.example:
|
|
||||||
CAMERA (allow):
|
|
||||||
camera=[
|
|
||||||
Access: [fg-s] 2025-01-01 00:00:00.000 (-1s) duration=+2ms
|
|
||||||
]
|
|
||||||
RECORD_AUDIO (allow):
|
|
||||||
Running start at: +3s
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
camera = da.results[0]["permissions"][0]["entries"][0]
|
|
||||||
running = da.results[0]["permissions"][1]["entries"][0]
|
|
||||||
assert camera["attribution"] == "camera"
|
|
||||||
assert running["event"] == "running"
|
|
||||||
assert running["relative_time"] == "+3s"
|
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
da = DumpsysAppopsArtifact()
|
da = DumpsysAppopsArtifact()
|
||||||
|
|||||||
@@ -57,19 +57,18 @@ class TestDumpsysBatteryDailyArtifact:
|
|||||||
assert uninstall_alert.message == (
|
assert uninstall_alert.message == (
|
||||||
"Detected uninstall of package com.example.removed (vers 0)"
|
"Detected uninstall of package com.example.removed (vers 0)"
|
||||||
)
|
)
|
||||||
assert uninstall_alert.event_time == "2022-08-16 15:56:39"
|
assert uninstall_alert.event_time == "2022-08-16"
|
||||||
assert uninstall_alert.event["package_name"] == "com.example.removed"
|
assert uninstall_alert.event["package_name"] == "com.example.removed"
|
||||||
assert uninstall_alert.event["version_code"] == 0
|
assert uninstall_alert.event["vers"] == "0"
|
||||||
assert uninstall_alert.event["action"] == "uninstall"
|
|
||||||
|
|
||||||
assert downgrade_alert.level == AlertLevel.MEDIUM
|
assert downgrade_alert.level == AlertLevel.MEDIUM
|
||||||
assert downgrade_alert.message == (
|
assert downgrade_alert.message == (
|
||||||
"Detected downgrade of package com.example.app from vers 10 to vers 9"
|
"Detected downgrade of package com.example.app from vers 10 to vers 9"
|
||||||
)
|
)
|
||||||
assert downgrade_alert.event_time == "2022-08-17 15:56:39"
|
assert downgrade_alert.event_time == "2022-08-17"
|
||||||
assert downgrade_alert.event["package_name"] == "com.example.app"
|
assert downgrade_alert.event["package_name"] == "com.example.app"
|
||||||
assert downgrade_alert.event["action"] == "downgrade"
|
assert downgrade_alert.event["action"] == "downgrade"
|
||||||
assert downgrade_alert.event["previous_version_code"] == 10
|
assert downgrade_alert.event["previous_vers"] == "10"
|
||||||
|
|
||||||
def test_newest_first_update_is_not_reported_as_downgrade(self):
|
def test_newest_first_update_is_not_reported_as_downgrade(self):
|
||||||
dba = DumpsysBatteryDailyArtifact()
|
dba = DumpsysBatteryDailyArtifact()
|
||||||
@@ -108,19 +107,7 @@ class TestDumpsysBatteryDailyArtifact:
|
|||||||
assert downgrade_alert.event_time == "2026-01-10"
|
assert downgrade_alert.event_time == "2026-01-10"
|
||||||
assert downgrade_alert.event["package_name"] == "com.example.app"
|
assert downgrade_alert.event["package_name"] == "com.example.app"
|
||||||
assert downgrade_alert.event["action"] == "downgrade"
|
assert downgrade_alert.event["action"] == "downgrade"
|
||||||
assert downgrade_alert.event["previous_version_code"] == 102
|
assert downgrade_alert.event["previous_vers"] == "102"
|
||||||
|
|
||||||
def test_duplicate_updates_retain_occurrence_count(self):
|
|
||||||
dba = DumpsysBatteryDailyArtifact()
|
|
||||||
dba.parse(
|
|
||||||
""" Daily from 2026-01-10-01-02-03 to 2026-01-11-04-05-06:
|
|
||||||
Update com.example.app vers=12
|
|
||||||
Update com.example.app vers=12
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
assert dba.results[0]["occurrences"] == 2
|
|
||||||
assert dba.results[0]["period_start"] == "2026-01-10 01:02:03"
|
|
||||||
|
|
||||||
def test_reinstall_after_uninstall_is_not_reported_as_downgrade(self):
|
def test_reinstall_after_uninstall_is_not_reported_as_downgrade(self):
|
||||||
dba = DumpsysBatteryDailyArtifact()
|
dba = DumpsysBatteryDailyArtifact()
|
||||||
|
|||||||
@@ -54,7 +54,6 @@ class TestDumpsysBatteryHistoryArtifact:
|
|||||||
assert len(dba.results) == 2
|
assert len(dba.results) == 2
|
||||||
assert dba.results[0] == {
|
assert dba.results[0] == {
|
||||||
"time_elapsed": "07-15 20:27:39.431",
|
"time_elapsed": "07-15 20:27:39.431",
|
||||||
"timestamp": "1900-07-15 20:27:39.431000",
|
|
||||||
"event": "start_job",
|
"event": "start_job",
|
||||||
"uid": "u0a123",
|
"uid": "u0a123",
|
||||||
"package_name": "com.example",
|
"package_name": "com.example",
|
||||||
@@ -62,23 +61,3 @@ class TestDumpsysBatteryHistoryArtifact:
|
|||||||
}
|
}
|
||||||
assert dba.results[1]["event"] == "end_job"
|
assert dba.results[1]["event"] == "end_job"
|
||||||
assert dba.results[1]["uid"] == "u0a123"
|
assert dba.results[1]["uid"] == "u0a123"
|
||||||
|
|
||||||
def test_wake_lock_without_component_is_retained(self):
|
|
||||||
dba = DumpsysBatteryHistoryArtifact()
|
|
||||||
dba.parse(
|
|
||||||
"Battery History:\n"
|
|
||||||
" 0 (2) 100 RESET:TIME: 2025-09-05-01-04-52-139\n"
|
|
||||||
' +1s (2) 100 +running +wake_lock=1000:"*alarm*:TIME_TICK"\n'
|
|
||||||
' +2s (2) 100 +running +wake_lock=u0a1:"*walarm*:com.whatsapp.MessageHandler.LOGOUT_ACTION"\n'
|
|
||||||
"\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert [record["event"] for record in dba.results] == ["wake", "wake"]
|
|
||||||
assert dba.results[0]["package_name"] is None
|
|
||||||
assert dba.results[1]["package_name"] == "com.whatsapp"
|
|
||||||
|
|
||||||
def test_decorated_sync_job_uses_component_package(self):
|
|
||||||
dba = DumpsysBatteryHistoryArtifact()
|
|
||||||
dba.parse('+1s (2) 100 +job=u0a1:"@SyncManager@gmail-ls/com.google:android"\n')
|
|
||||||
|
|
||||||
assert dba.results[0]["package_name"] == "com.google"
|
|
||||||
@@ -53,32 +53,9 @@ Connection pool for /data/user/0/com.example/databases/current.db:
|
|||||||
|
|
||||||
assert dbi.results == [
|
assert dbi.results == [
|
||||||
{
|
{
|
||||||
"timestamp": "07-15 20:27:39.431",
|
"isodate": "07-15 20:27:39.431",
|
||||||
"pid": None,
|
|
||||||
"action": "executeForCursorWindow",
|
"action": "executeForCursorWindow",
|
||||||
"duration_ms": 1,
|
|
||||||
"status": "succeeded",
|
|
||||||
"sql": "SELECT 1",
|
"sql": "SELECT 1",
|
||||||
"path": "/data/user/0/com.example/databases/current.db",
|
"path": "/data/user/0/com.example/databases/current.db",
|
||||||
"pool_path": "/data/user/0/com.example/databases/current.db",
|
|
||||||
"connection_number": None,
|
|
||||||
"is_primary": None,
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
def test_parses_operations_from_multiple_connections(self):
|
|
||||||
dbi = DumpsysDBInfoArtifact()
|
|
||||||
dbi.parse(
|
|
||||||
"""Connection pool for /data/example.db:
|
|
||||||
Connection #0:
|
|
||||||
isPrimaryConnection: true
|
|
||||||
Most recently executed operations:
|
|
||||||
0: [2025-01-01 00:00:00.000] execute took 1ms - succeeded, sql="SELECT 1", path=/data/example.db
|
|
||||||
Connection #1:
|
|
||||||
isPrimaryConnection: false
|
|
||||||
Most recently executed operations:
|
|
||||||
0: [2025-01-01 00:00:01.000] execute took 2ms - succeeded, sql="SELECT 2", path=/data/example.db
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
assert [record["connection_number"] for record in dbi.results] == [0, 1]
|
|
||||||
@@ -21,16 +21,12 @@ class TestDumpsysPackageActivitiesArtifact:
|
|||||||
|
|
||||||
assert len(dpa.results) == 0
|
assert len(dpa.results) == 0
|
||||||
dpa.parse(data)
|
dpa.parse(data)
|
||||||
assert len(dpa.results) == 10
|
assert len(dpa.results) == 4
|
||||||
assert dpa.results[0]["package_name"] == "com.samsung.android.messaging"
|
assert dpa.results[0]["package_name"] == "com.samsung.android.app.social"
|
||||||
assert (
|
assert (
|
||||||
dpa.results[0]["component"]
|
dpa.results[0]["activity"]
|
||||||
== "com.samsung.android.messaging/.ui.RcsTransferContent"
|
== "com.samsung.android.app.social/.feed.FeedsActivity"
|
||||||
)
|
)
|
||||||
assert {result["resolver_type"] for result in dpa.results} == {
|
|
||||||
"full_mime_type",
|
|
||||||
"non_data_action",
|
|
||||||
}
|
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
dpa = DumpsysPackageActivitiesArtifact()
|
dpa = DumpsysPackageActivitiesArtifact()
|
||||||
@@ -45,4 +41,4 @@ class TestDumpsysPackageActivitiesArtifact:
|
|||||||
dpa.indicators = ind
|
dpa.indicators = ind
|
||||||
assert len(dpa.alertstore.alerts) == 0
|
assert len(dpa.alertstore.alerts) == 0
|
||||||
dpa.check_indicators()
|
dpa.check_indicators()
|
||||||
assert len(dpa.alertstore.alerts) == 2
|
assert len(dpa.alertstore.alerts) == 1
|
||||||
@@ -25,11 +25,7 @@ class TestDumpsysPackagesArtifact:
|
|||||||
== "com.samsung.android.provider.filterprovider"
|
== "com.samsung.android.provider.filterprovider"
|
||||||
)
|
)
|
||||||
assert dpa.results[0]["version_name"] == "5.0.07"
|
assert dpa.results[0]["version_name"] == "5.0.07"
|
||||||
assert dpa.results[0]["version_code"] == 500700000
|
assert dpa.results[0]["first_install_time"] == "2008-12-31 16:00:00"
|
||||||
assert dpa.results[0]["min_sdk"] == 28
|
|
||||||
assert dpa.results[0]["target_sdk"] == 28
|
|
||||||
assert dpa.results[0]["package_type"] == "active"
|
|
||||||
assert dpa.results[0]["users"][0]["user_id"] == 0
|
|
||||||
assert dpa.results[0]["system"] is True
|
assert dpa.results[0]["system"] is True
|
||||||
|
|
||||||
def test_parsing_system_flag(self):
|
def test_parsing_system_flag(self):
|
||||||
@@ -64,27 +60,42 @@ class TestDumpsysPackagesArtifact:
|
|||||||
dpa.check_indicators()
|
dpa.check_indicators()
|
||||||
assert len(dpa.alertstore.alerts) == 1
|
assert len(dpa.alertstore.alerts) == 1
|
||||||
|
|
||||||
def test_hidden_packages_and_per_user_state(self):
|
def test_per_user_fields_use_primary_user(self):
|
||||||
dpa = DumpsysPackagesArtifact()
|
details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
dpa.parse(
|
""" User 0: installed=true
|
||||||
"""Packages:
|
firstInstallTime=2024-01-10 09:19:39
|
||||||
Package [com.example.active]:
|
runtime permissions:
|
||||||
appId=10001
|
android.permission.CAMERA: granted=true
|
||||||
versionCode=12 minSdk=29 targetSdk=35
|
User 95: installed=false
|
||||||
User 0: installed=true hidden=false
|
firstInstallTime=1970-01-01 01:00:00
|
||||||
firstInstallTime=2025-01-01 01:02:03
|
runtime permissions:
|
||||||
User 10: installed=false hidden=true
|
android.permission.CAMERA: granted=false
|
||||||
firstInstallTime=2025-01-02 01:02:03
|
android.permission.RECORD_AUDIO: granted=false
|
||||||
Hidden system packages:
|
|
||||||
Package [com.example.hidden]:
|
|
||||||
appId=10002
|
|
||||||
installerPackageName=null
|
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
assert [record["package_type"] for record in dpa.results] == [
|
assert details["first_install_time"] == "2024-01-10 09:19:39"
|
||||||
"active",
|
runtime_permissions = [
|
||||||
"hidden_system",
|
permission
|
||||||
|
for permission in details["permissions"]
|
||||||
|
if permission["type"] == "runtime"
|
||||||
]
|
]
|
||||||
assert len(dpa.results[0]["users"]) == 2
|
assert runtime_permissions == [
|
||||||
assert dpa.results[1]["installer"] is None
|
{
|
||||||
|
"name": "android.permission.CAMERA",
|
||||||
|
"granted": True,
|
||||||
|
"type": "runtime",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_per_user_fields_fall_back_when_user_zero_is_missing(self):
|
||||||
|
details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
|
""" User 10: installed=true
|
||||||
|
firstInstallTime=2024-02-10 09:19:39
|
||||||
|
runtime permissions:
|
||||||
|
android.permission.CAMERA: granted=true
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert details["first_install_time"] == "2024-02-10 09:19:39"
|
||||||
|
assert details["permissions"][-1]["name"] == "android.permission.CAMERA"
|
||||||
@@ -22,9 +22,6 @@ class TestDumpsysPlatformCompatArtifact:
|
|||||||
assert len(dbi.results) == 2
|
assert len(dbi.results) == 2
|
||||||
assert dbi.results[0]["package_name"] == "org.torproject.torbrowser"
|
assert dbi.results[0]["package_name"] == "org.torproject.torbrowser"
|
||||||
assert dbi.results[1]["package_name"] == "org.article19.circulo.next"
|
assert dbi.results[1]["package_name"] == "org.article19.circulo.next"
|
||||||
assert dbi.results[0]["change_id"] == 168419799
|
|
||||||
assert dbi.results[0]["change_name"] == "DOWNSCALED"
|
|
||||||
assert dbi.results[0]["override_value"] is False
|
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
dbi = DumpsysPlatformCompatArtifact()
|
dbi = DumpsysPlatformCompatArtifact()
|
||||||
|
|||||||
@@ -19,14 +19,17 @@ class TestDumpsysReceiversArtifact:
|
|||||||
|
|
||||||
assert len(dr.results) == 0
|
assert len(dr.results) == 0
|
||||||
dr.parse(data)
|
dr.parse(data)
|
||||||
assert len(dr.results) == 9
|
assert len(dr.results) == 4
|
||||||
assert dr.results[0]["resolver_type"] == "full_mime_type"
|
assert (
|
||||||
storage_manager = next(
|
list(dr.results.keys())[0]
|
||||||
result
|
== "com.android.storagemanager.automatic.SHOW_NOTIFICATION"
|
||||||
for result in dr.results
|
)
|
||||||
if result["key"] == "com.android.storagemanager.automatic.SHOW_NOTIFICATION"
|
assert (
|
||||||
|
dr.results["com.android.storagemanager.automatic.SHOW_NOTIFICATION"][0][
|
||||||
|
"package_name"
|
||||||
|
]
|
||||||
|
== "com.android.storagemanager"
|
||||||
)
|
)
|
||||||
assert storage_manager["package_name"] == "com.android.storagemanager"
|
|
||||||
|
|
||||||
def test_parsing_misindented_action(self):
|
def test_parsing_misindented_action(self):
|
||||||
dr = DumpsysReceiversArtifact()
|
dr = DumpsysReceiversArtifact()
|
||||||
@@ -41,8 +44,12 @@ Receiver Resolver Table:
|
|||||||
|
|
||||||
dr.parse(data)
|
dr.parse(data)
|
||||||
|
|
||||||
assert dr.results[1]["key"] == "android.intent.action.MY_PACKAGE_REPLACED"
|
assert (
|
||||||
assert dr.results[1]["package_name"] == "com.psycatgames.nhiegame"
|
dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][
|
||||||
|
"package_name"
|
||||||
|
]
|
||||||
|
== "com.psycatgames.nhiegame"
|
||||||
|
)
|
||||||
|
|
||||||
def test_parsing_misindented_first_action(self):
|
def test_parsing_misindented_first_action(self):
|
||||||
dr = DumpsysReceiversArtifact()
|
dr = DumpsysReceiversArtifact()
|
||||||
@@ -55,8 +62,12 @@ Receiver Resolver Table:
|
|||||||
|
|
||||||
dr.parse(data)
|
dr.parse(data)
|
||||||
|
|
||||||
assert dr.results[0]["key"] == "android.intent.action.MY_PACKAGE_REPLACED"
|
assert (
|
||||||
assert dr.results[0]["package_name"] == "com.psycatgames.nhiegame"
|
dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][
|
||||||
|
"package_name"
|
||||||
|
]
|
||||||
|
== "com.psycatgames.nhiegame"
|
||||||
|
)
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
dr = DumpsysReceiversArtifact()
|
dr = DumpsysReceiversArtifact()
|
||||||
|
|||||||
@@ -39,29 +39,3 @@ class TestGetPropArtifact:
|
|||||||
assert len(gp.alertstore.alerts) == 0
|
assert len(gp.alertstore.alerts) == 0
|
||||||
gp.check_indicators()
|
gp.check_indicators()
|
||||||
assert len(gp.alertstore.alerts) == 1
|
assert len(gp.alertstore.alerts) == 1
|
||||||
|
|
||||||
def test_empty_values_and_invalid_lines(self):
|
|
||||||
gp = GetProp()
|
|
||||||
gp.parse("[empty]: []\n[valid]: [value]\n0\n[broken]: [value")
|
|
||||||
|
|
||||||
assert gp.results == [
|
|
||||||
{"name": "empty", "value": ""},
|
|
||||||
{"name": "valid", "value": "value"},
|
|
||||||
]
|
|
||||||
|
|
||||||
def test_multiline_value(self):
|
|
||||||
gp = GetProp()
|
|
||||||
gp.parse(
|
|
||||||
"[persist.sys.boot.reason.history]: ["
|
|
||||||
"reboot,ota,1697044974\n"
|
|
||||||
"reboot,watchdog,1696958574]\n"
|
|
||||||
"[ro.build.version.sdk]: [35]\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert gp.results == [
|
|
||||||
{
|
|
||||||
"name": "persist.sys.boot.reason.history",
|
|
||||||
"value": "reboot,ota,1697044974\nreboot,watchdog,1696958574",
|
|
||||||
},
|
|
||||||
{"name": "ro.build.version.sdk", "value": "35"},
|
|
||||||
]
|
|
||||||
@@ -20,7 +20,7 @@ class TestProcessesArtifact:
|
|||||||
assert len(p.results) == 0
|
assert len(p.results) == 0
|
||||||
p.parse(data)
|
p.parse(data)
|
||||||
assert len(p.results) == 17
|
assert len(p.results) == 17
|
||||||
assert p.results[0]["command"] == "init"
|
assert p.results[0]["proc_name"] == "init"
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
p = Processes()
|
p = Processes()
|
||||||
@@ -36,14 +36,3 @@ class TestProcessesArtifact:
|
|||||||
assert len(p.alertstore.alerts) == 0
|
assert len(p.alertstore.alerts) == 0
|
||||||
p.check_indicators()
|
p.check_indicators()
|
||||||
assert len(p.alertstore.alerts) == 1
|
assert len(p.alertstore.alerts) == 1
|
||||||
|
|
||||||
def test_bugreport_thread_columns(self):
|
|
||||||
p = Processes()
|
|
||||||
p.parse(
|
|
||||||
"LABEL USER PID TID PPID VSZ RSS WCHAN ADDR S PRI NI RTPRIO SCH PCY TIME CMD\n"
|
|
||||||
"u:r:init:s0 root 1 2 0 100 20 0 0 S 19 0 - 0 fg 00:00:01 init\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert p.results[0]["label"] == "u:r:init:s0"
|
|
||||||
assert p.results[0]["tid"] == 2
|
|
||||||
assert p.results[0]["command"] == "init"
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2023 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
"""A `Caused by:` line must not discard the whole text tombstone.
|
|
||||||
|
|
||||||
Keys are matched as bare prefixes, so `Caused by: …` — an ordinary line inside
|
|
||||||
an abort message — reached the `Cause` key, failed the key comparison and
|
|
||||||
raised, which `Tombstones.run()` logged while dropping the entire crash record.
|
|
||||||
Seen on a 1.6 MB tombstone whose protobuf twin was zero bytes: the crash then
|
|
||||||
had no representation at all.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import datetime
|
|
||||||
|
|
||||||
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
|
|
||||||
|
|
||||||
TOMBSTONE = b"""\
|
|
||||||
*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
|
|
||||||
Build fingerprint: 'Xiaomi/vili_eea/vili:13/TKQ1.220829.002/V14.0.10.0:user/release-keys'
|
|
||||||
Revision: '0'
|
|
||||||
ABI: 'arm64'
|
|
||||||
Timestamp: 2023-08-24 14:54:47.999124034+0300
|
|
||||||
Process uptime: 12199s
|
|
||||||
Cmdline: com.example.game
|
|
||||||
pid: 8044, tid: 26222, name: UnityMain >>> com.example.game <<<
|
|
||||||
uid: 10235
|
|
||||||
signal 6 (SIGABRT), code -1 (SI_QUEUE), fault addr --------
|
|
||||||
Abort message: 'No pending exception expected: java.lang.SecurityException: listen
|
|
||||||
at void android.os.Parcel.readException() (Parcel.java:2920)
|
|
||||||
Caused by: android.os.RemoteException: Remote stack trace:
|
|
||||||
\tat com.android.server.TelephonyRegistry.listen(TelephonyRegistry.java:1096)
|
|
||||||
"""
|
|
||||||
|
|
||||||
WITH_CAUSE = TOMBSTONE + b"Cause: null pointer dereference\n"
|
|
||||||
|
|
||||||
|
|
||||||
class TestTombstoneCausedBy:
|
|
||||||
def _parse(self, content):
|
|
||||||
artifact = TombstoneCrashArtifact()
|
|
||||||
artifact.results = []
|
|
||||||
artifact.parse("tombstone_23", datetime.datetime(2023, 8, 24), content)
|
|
||||||
return artifact.results
|
|
||||||
|
|
||||||
def test_caused_by_line_does_not_discard_the_tombstone(self):
|
|
||||||
results = self._parse(TOMBSTONE)
|
|
||||||
assert len(results) == 1
|
|
||||||
assert results[0]["pid"] == 8044
|
|
||||||
assert results[0]["process_name"] == "UnityMain"
|
|
||||||
assert results[0]["uid"] == 10235
|
|
||||||
|
|
||||||
def test_the_real_cause_key_is_still_parsed(self):
|
|
||||||
results = self._parse(WITH_CAUSE)
|
|
||||||
assert results[0]["cause"] == "null pointer dereference"
|
|
||||||
@@ -128,5 +128,8 @@ class TestTombstoneCrashArtifact:
|
|||||||
assert tombstone_result.get("pid") == 25541
|
assert tombstone_result.get("pid") == 25541
|
||||||
assert tombstone_result.get("process_name") == "mtk.ape.decoder"
|
assert tombstone_result.get("process_name") == "mtk.ape.decoder"
|
||||||
|
|
||||||
# Tombstones include an explicit offset, so normalize them to UTC.
|
# With Android logs we want to keep timestamps as device local time for consistency.
|
||||||
assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.518290"
|
# We often don't know the time offset for a log entry and so can't convert everything to UTC.
|
||||||
|
# MVT should output the local time only:
|
||||||
|
# So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000
|
||||||
|
assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.518290"
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2023 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
"""An encrypted backup.ab must not take the whole check-androidqf run with it.
|
|
||||||
|
|
||||||
`CmdAndroidCheckBackup.from_ab()` already raises `InvalidAndroidBackup` instead
|
|
||||||
of exiting when it runs as a sub-command (`check-androidqf` catches that and
|
|
||||||
skips the backup modules), for a wrong file format and for a parse error. The
|
|
||||||
password branches used to call `sys.exit(1)` unconditionally, which ends the
|
|
||||||
parent run inside `finish()` — before the intrusion-logs command and before the
|
|
||||||
timeline, alerts, urls, info and run-manifest are written.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from mvt.android.cmd_check_backup import CmdAndroidCheckBackup, InvalidAndroidBackup
|
|
||||||
|
|
||||||
ENCRYPTED_AB_HEADER = b"ANDROID BACKUP\n5\n0\nAES-256\n" + b"\x00" * 64
|
|
||||||
|
|
||||||
|
|
||||||
class TestCheckBackupOptionalFailure:
|
|
||||||
def _cmd(self, tmp_path, sub_command):
|
|
||||||
return CmdAndroidCheckBackup(
|
|
||||||
target_path=None,
|
|
||||||
results_path=str(tmp_path),
|
|
||||||
module_options={"interactive": False},
|
|
||||||
sub_command=sub_command,
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_missing_password_raises_when_nested(self, tmp_path):
|
|
||||||
cmd = self._cmd(tmp_path, sub_command=True)
|
|
||||||
with pytest.raises(InvalidAndroidBackup):
|
|
||||||
cmd.from_ab(ENCRYPTED_AB_HEADER)
|
|
||||||
|
|
||||||
def test_missing_password_still_exits_on_its_own_command(self, tmp_path):
|
|
||||||
cmd = self._cmd(tmp_path, sub_command=False)
|
|
||||||
with pytest.raises(SystemExit):
|
|
||||||
cmd.from_ab(ENCRYPTED_AB_HEADER)
|
|
||||||
@@ -13,19 +13,6 @@ from ..utils import get_android_androidqf, list_files
|
|||||||
|
|
||||||
|
|
||||||
class TestAndroidqfMountsArtifact:
|
class TestAndroidqfMountsArtifact:
|
||||||
def test_parse_proc_mountinfo(self):
|
|
||||||
from mvt.android.artifacts.mounts import Mounts as MountsArtifact
|
|
||||||
|
|
||||||
results = MountsArtifact.parse_mountinfo(
|
|
||||||
"41 40 254:13 / / ro,relatime shared:1 - erofs /dev/block/dm-13 ro,seclabel\n",
|
|
||||||
123,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert results[0]["mount_id"] == 41
|
|
||||||
assert results[0]["mount_point"] == "/"
|
|
||||||
assert results[0]["filesystem_type"] == "erofs"
|
|
||||||
assert results[0]["process_ids"] == [123]
|
|
||||||
|
|
||||||
def test_parse_mounts_token_checks(self):
|
def test_parse_mounts_token_checks(self):
|
||||||
"""
|
"""
|
||||||
Test the artifact-level `parse` method using tolerant token checks.
|
Test the artifact-level `parse` method using tolerant token checks.
|
||||||
|
|||||||
@@ -6,27 +6,12 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from mvt.android.modules.androidqf.aqf_settings import AQFSettings
|
from mvt.android.modules.androidqf.aqf_settings import AQFSettings
|
||||||
from mvt.android.artifacts.settings import Settings
|
|
||||||
from mvt.common.module import run_module
|
from mvt.common.module import run_module
|
||||||
|
|
||||||
from ..utils import get_android_androidqf, list_files
|
from ..utils import get_android_androidqf, list_files
|
||||||
|
|
||||||
|
|
||||||
class TestSettingsModule:
|
class TestSettingsModule:
|
||||||
def test_bugreport_settings_format(self):
|
|
||||||
settings = Settings()
|
|
||||||
settings.parse(
|
|
||||||
"GLOBAL SETTINGS (user 0)\n"
|
|
||||||
"_id:1 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true\n"
|
|
||||||
"SECURE SETTINGS (user 10)\n"
|
|
||||||
"_id:2 name:accessibility_enabled pkg:android value:1\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert settings.results == {
|
|
||||||
"global:user_0": {"adb_wifi_enabled": "0"},
|
|
||||||
"secure:user_10": {"accessibility_enabled": "1"},
|
|
||||||
}
|
|
||||||
|
|
||||||
def test_parsing(self):
|
def test_parsing(self):
|
||||||
data_path = get_android_androidqf()
|
data_path = get_android_androidqf()
|
||||||
m = AQFSettings(target_path=data_path)
|
m = AQFSettings(target_path=data_path)
|
||||||
|
|||||||
@@ -54,11 +54,10 @@ class TestBugreportAnalysis:
|
|||||||
== "com.samsung.android.provider.filterprovider"
|
== "com.samsung.android.provider.filterprovider"
|
||||||
)
|
)
|
||||||
assert m.results[1]["package_name"] == "com.instagram.android"
|
assert m.results[1]["package_name"] == "com.instagram.android"
|
||||||
assert m.results[0]["installer"] is None
|
assert m.results[0]["installer"] == ""
|
||||||
assert m.results[1]["installer"] == "com.android.vending"
|
assert m.results[1]["installer"] == "com.android.vending"
|
||||||
assert len(m.results[0]["permissions"]) == 4
|
assert len(m.results[0]["permissions"]) == 4
|
||||||
assert len(m.results[1]["permissions"]) == 20
|
assert len(m.results[1]["permissions"]) == 32
|
||||||
assert len(m.results[1]["users"][0]["permissions"]) == 19
|
|
||||||
|
|
||||||
def test_getprop_module(self):
|
def test_getprop_module(self):
|
||||||
m = self.launch_bug_report_module(DumpsysGetProp)
|
m = self.launch_bug_report_module(DumpsysGetProp)
|
||||||
@@ -67,34 +66,29 @@ class TestBugreportAnalysis:
|
|||||||
def test_receivers_match_exact_package_name(self, indicators_factory):
|
def test_receivers_match_exact_package_name(self, indicators_factory):
|
||||||
intent = "android.intent.action.PHONE_STATE"
|
intent = "android.intent.action.PHONE_STATE"
|
||||||
false_positive = {
|
false_positive = {
|
||||||
"resolver_type": "non_data_action",
|
|
||||||
"key": intent,
|
|
||||||
"package_name": "com.android.phone",
|
"package_name": "com.android.phone",
|
||||||
"component": (
|
"receiver": (
|
||||||
"com.android.phone/"
|
"com.android.phone/"
|
||||||
"com.android.services.telephony.sip.SipIncomingCallReceiver"
|
"com.android.services.telephony.sip.SipIncomingCallReceiver"
|
||||||
),
|
),
|
||||||
"filter_count": 1,
|
|
||||||
}
|
}
|
||||||
malicious_receiver = {
|
malicious_receiver = {
|
||||||
"resolver_type": "non_data_action",
|
|
||||||
"key": intent,
|
|
||||||
"package_name": "com.android.services",
|
"package_name": "com.android.services",
|
||||||
"component": "com.android.services/com.example.SomeReceiver",
|
"receiver": "com.android.services/com.example.SomeReceiver",
|
||||||
"filter_count": 1,
|
|
||||||
}
|
}
|
||||||
module = DumpsysReceivers(results=[false_positive, malicious_receiver])
|
module = DumpsysReceivers(
|
||||||
|
results={intent: [false_positive, malicious_receiver]}
|
||||||
|
)
|
||||||
module.indicators = indicators_factory(app_ids=["com.android.services"])
|
module.indicators = indicators_factory(app_ids=["com.android.services"])
|
||||||
|
|
||||||
module.check_indicators()
|
module.check_indicators()
|
||||||
|
|
||||||
assert len(module.alertstore.alerts) == 1
|
assert len(module.alertstore.alerts) == 1
|
||||||
alert = module.alertstore.alerts[0]
|
alert = module.alertstore.alerts[0]
|
||||||
assert alert.event == malicious_receiver
|
assert alert.event == {intent: malicious_receiver}
|
||||||
assert alert.matched_indicator.value == "com.android.services"
|
assert alert.matched_indicator.value == "com.android.services"
|
||||||
|
|
||||||
def test_tombstones_modules(self):
|
def test_tombstones_modules(self):
|
||||||
m = self.launch_bug_report_module(Tombstones)
|
m = self.launch_bug_report_module(Tombstones)
|
||||||
assert len(m.results) == 2
|
assert len(m.results) == 2
|
||||||
assert m.results[1]["pid"] == 3559
|
assert m.results[1]["pid"] == 3559
|
||||||
assert m.results[0]["sources"]["text"]["parsed"] is True
|
|
||||||
@@ -3,12 +3,9 @@ from types import SimpleNamespace
|
|||||||
import click
|
import click
|
||||||
from click.testing import CliRunner
|
from click.testing import CliRunner
|
||||||
|
|
||||||
from mvt.cli import cli as mvt_cli
|
|
||||||
from mvt.common.cli_plugins import (
|
from mvt.common.cli_plugins import (
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
ANDROID_CLI_PLUGIN_GROUP,
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
IOS_CLI_PLUGIN_GROUP,
|
||||||
MVT_CUSTOM_COMMANDS_ENV,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
BrokenPluginCommand,
|
BrokenPluginCommand,
|
||||||
load_cli_commands_option,
|
load_cli_commands_option,
|
||||||
register_cli_commands_from_path,
|
register_cli_commands_from_path,
|
||||||
@@ -17,9 +14,6 @@ from mvt.common.cli_plugins import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Keep the banner of the mvt group callback from checking for updates online.
|
|
||||||
OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"]
|
|
||||||
|
|
||||||
COMMAND_TEMPLATE = """
|
COMMAND_TEMPLATE = """
|
||||||
import click
|
import click
|
||||||
|
|
||||||
@@ -351,11 +345,7 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate(
|
|||||||
def entry_points(*, group):
|
def entry_points(*, group):
|
||||||
if group == IOS_CLI_PLUGIN_GROUP:
|
if group == IOS_CLI_PLUGIN_GROUP:
|
||||||
return [_entry_point("ios-package", "ios_plugin:cli", ios_package)]
|
return [_entry_point("ios-package", "ios_plugin:cli", ios_package)]
|
||||||
if group == ANDROID_CLI_PLUGIN_GROUP:
|
return [_entry_point("android-package", "android_plugin:cli", android_package)]
|
||||||
return [
|
|
||||||
_entry_point("android-package", "android_plugin:cli", android_package)
|
|
||||||
]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
@@ -379,187 +369,3 @@ def test_platform_entry_point_groups_and_environment_paths_are_separate(
|
|||||||
|
|
||||||
assert set(ios_group.commands) == {"ios-file", "ios-package"}
|
assert set(ios_group.commands) == {"ios-file", "ios-package"}
|
||||||
assert set(android_group.commands) == {"android-file", "android-package"}
|
assert set(android_group.commands) == {"android-file", "android-package"}
|
||||||
|
|
||||||
|
|
||||||
def test_neutral_entry_point_group_is_not_registered_on_the_platform_clis(
|
|
||||||
monkeypatch,
|
|
||||||
):
|
|
||||||
@click.command()
|
|
||||||
def neutral_package():
|
|
||||||
pass
|
|
||||||
|
|
||||||
def entry_points(*, group):
|
|
||||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
|
||||||
return [
|
|
||||||
_entry_point("neutral-package", "neutral_plugin:cli", neutral_package)
|
|
||||||
]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
|
||||||
entry_points,
|
|
||||||
)
|
|
||||||
ios_group = click.Group()
|
|
||||||
android_group = click.Group()
|
|
||||||
|
|
||||||
register_cli_plugins(
|
|
||||||
ios_group,
|
|
||||||
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable="TEST_IOS_COMMANDS",
|
|
||||||
)
|
|
||||||
register_cli_plugins(
|
|
||||||
android_group,
|
|
||||||
entry_point_group=ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable="TEST_ANDROID_COMMANDS",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert not ios_group.commands
|
|
||||||
assert not android_group.commands
|
|
||||||
|
|
||||||
|
|
||||||
def test_environment_command_wins_collision_with_installed_command(
|
|
||||||
tmp_path, monkeypatch, caplog
|
|
||||||
):
|
|
||||||
command_path = _write_command(
|
|
||||||
tmp_path / "duplicate.py",
|
|
||||||
"duplicate",
|
|
||||||
message="environment command ran",
|
|
||||||
)
|
|
||||||
|
|
||||||
@click.command()
|
|
||||||
def installed_command():
|
|
||||||
pass
|
|
||||||
|
|
||||||
def entry_points(*, group):
|
|
||||||
if group == IOS_CLI_PLUGIN_GROUP:
|
|
||||||
return [
|
|
||||||
_entry_point(
|
|
||||||
"duplicate",
|
|
||||||
"ios_plugin:cli",
|
|
||||||
installed_command,
|
|
||||||
distribution="ios-plugin",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
|
||||||
entry_points,
|
|
||||||
)
|
|
||||||
monkeypatch.setenv("TEST_IOS_COMMANDS", str(command_path))
|
|
||||||
group = click.Group()
|
|
||||||
|
|
||||||
register_cli_plugins(
|
|
||||||
group,
|
|
||||||
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable="TEST_IOS_COMMANDS",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert group.commands["duplicate"] is not installed_command
|
|
||||||
result = CliRunner().invoke(group, ["duplicate"])
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert "environment command ran" in result.output
|
|
||||||
assert "the command name is already registered" in caplog.text
|
|
||||||
assert "ios-plugin 1.0 (ios_plugin:cli)" in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_mvt_cli_gets_the_neutral_commands_and_no_platform_command(
|
|
||||||
monkeypatch, restore_cli_commands
|
|
||||||
):
|
|
||||||
@click.command()
|
|
||||||
def shared_package():
|
|
||||||
click.echo("shared command ran")
|
|
||||||
|
|
||||||
@click.command()
|
|
||||||
def ios_package():
|
|
||||||
pass
|
|
||||||
|
|
||||||
def entry_points(*, group):
|
|
||||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
|
||||||
return [_entry_point("shared-package", "shared_plugin:cli", shared_package)]
|
|
||||||
if group == IOS_CLI_PLUGIN_GROUP:
|
|
||||||
return [_entry_point("ios-package", "ios_plugin:cli", ios_package)]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
|
||||||
entry_points,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cli_plugins(
|
|
||||||
mvt_cli,
|
|
||||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert "ios-package" not in mvt_cli.commands
|
|
||||||
result = CliRunner().invoke(mvt_cli, [*OFFLINE, "shared-package"])
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert "shared command ran" in result.output
|
|
||||||
|
|
||||||
|
|
||||||
def test_builtin_mvt_command_wins_collision_with_neutral_command(
|
|
||||||
monkeypatch, caplog, restore_cli_commands
|
|
||||||
):
|
|
||||||
@click.command()
|
|
||||||
def neutral_version():
|
|
||||||
pass
|
|
||||||
|
|
||||||
def entry_points(*, group):
|
|
||||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
|
||||||
return [
|
|
||||||
_entry_point(
|
|
||||||
"version",
|
|
||||||
"neutral_plugin:cli",
|
|
||||||
neutral_version,
|
|
||||||
distribution="neutral-plugin",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
|
||||||
entry_points,
|
|
||||||
)
|
|
||||||
builtin_version = mvt_cli.commands["version"]
|
|
||||||
|
|
||||||
register_cli_plugins(
|
|
||||||
mvt_cli,
|
|
||||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert mvt_cli.commands["version"] is builtin_version
|
|
||||||
assert "the command name is already registered" in caplog.text
|
|
||||||
assert "neutral-plugin 1.0 (neutral_plugin:cli)" in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_broken_neutral_plugin_does_not_break_the_mvt_cli(
|
|
||||||
monkeypatch, restore_cli_commands
|
|
||||||
):
|
|
||||||
def entry_points(*, group):
|
|
||||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
|
||||||
return [
|
|
||||||
_entry_point(
|
|
||||||
"broken",
|
|
||||||
"broken_plugin:cli",
|
|
||||||
exception=RuntimeError("missing dependency"),
|
|
||||||
distribution="broken-plugin",
|
|
||||||
)
|
|
||||||
]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
|
||||||
entry_points,
|
|
||||||
)
|
|
||||||
|
|
||||||
register_cli_plugins(
|
|
||||||
mvt_cli,
|
|
||||||
entry_point_group=NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
environment_variable=MVT_CUSTOM_COMMANDS_ENV,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert isinstance(mvt_cli.commands["broken"], BrokenPluginCommand)
|
|
||||||
result = CliRunner().invoke(mvt_cli, [*OFFLINE, "version"])
|
|
||||||
assert result.exit_code == 0
|
|
||||||
@@ -1,225 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from click.testing import CliRunner
|
|
||||||
|
|
||||||
from mvt.android.cli import cli as android_cli
|
|
||||||
from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES
|
|
||||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
from mvt.ios.cli import cli as ios_cli
|
|
||||||
from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES
|
|
||||||
from mvt.ios.modules.backup.manifest import Manifest
|
|
||||||
|
|
||||||
# Keep the banner of the group callback from checking for updates online.
|
|
||||||
OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"]
|
|
||||||
|
|
||||||
|
|
||||||
class CustomResultsModule(MVTModule):
|
|
||||||
"""A custom module which declares the check-iocs pair of both platforms."""
|
|
||||||
|
|
||||||
slug = "custom_results"
|
|
||||||
supported_commands = (
|
|
||||||
("ios", "check-backup"),
|
|
||||||
("ios", "check-iocs"),
|
|
||||||
("android", "check-iocs"),
|
|
||||||
)
|
|
||||||
|
|
||||||
checked: list = []
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
|
||||||
self.checked.append(list(self.results))
|
|
||||||
|
|
||||||
|
|
||||||
class BackupCheckerModule(MVTModule):
|
|
||||||
"""An iOS module which implements check_indicators() without declaring check-iocs."""
|
|
||||||
|
|
||||||
slug = "backup_checker"
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
|
|
||||||
checked: list = []
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
|
||||||
self.checked.append(list(self.results))
|
|
||||||
|
|
||||||
|
|
||||||
class BugReportCheckerModule(MVTModule):
|
|
||||||
"""The same for Android."""
|
|
||||||
|
|
||||||
slug = "bugreport_checker"
|
|
||||||
supported_commands = (("android", "check-bugreport"),)
|
|
||||||
|
|
||||||
checked: list = []
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
|
||||||
self.checked.append(list(self.results))
|
|
||||||
|
|
||||||
|
|
||||||
class BackupOnlyModule(MVTModule):
|
|
||||||
"""A custom module which does not implement check_indicators()."""
|
|
||||||
|
|
||||||
slug = "backup_only"
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"platform, builtin_modules, checker_module",
|
|
||||||
[
|
|
||||||
("ios", IOS_CHECK_IOCS_MODULES, BackupCheckerModule),
|
|
||||||
("android", ANDROID_CHECK_IOCS_MODULES, BugReportCheckerModule),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_check_iocs_rechecks_the_stored_results_of_custom_modules(
|
|
||||||
platform, builtin_modules, checker_module, tmp_path, caplog
|
|
||||||
):
|
|
||||||
# check-iocs matches every <slug>.json in the results folder to the module
|
|
||||||
# with that slug, custom modules included, and runs its check_indicators()
|
|
||||||
# again over the stored results.
|
|
||||||
results = [{"domain": "example.org"}]
|
|
||||||
(tmp_path / "custom_results.json").write_text(json.dumps(results))
|
|
||||||
(tmp_path / f"{checker_module.slug}.json").write_text(json.dumps(results))
|
|
||||||
(tmp_path / "backup_only.json").write_text(json.dumps(results))
|
|
||||||
CustomResultsModule.checked.clear()
|
|
||||||
checker_module.checked.clear()
|
|
||||||
|
|
||||||
cmd = CmdCheckIOCS(
|
|
||||||
target_path=str(tmp_path),
|
|
||||||
custom_modules=[CustomResultsModule, checker_module, BackupOnlyModule],
|
|
||||||
platform=platform,
|
|
||||||
)
|
|
||||||
cmd.modules = builtin_modules
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
# A module which declares the check-iocs pair is re-checked.
|
|
||||||
assert CustomResultsModule.checked == [results]
|
|
||||||
assert (
|
|
||||||
'Loading results from "custom_results.json" with module CustomResultsModule'
|
|
||||||
in caplog.text
|
|
||||||
)
|
|
||||||
# So is a module which only implements check_indicators().
|
|
||||||
assert checker_module.checked == [results]
|
|
||||||
# A module which does neither is not part of check-iocs.
|
|
||||||
assert "backup_only.json" not in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"platform, builtin_modules, listed, not_listed",
|
|
||||||
[
|
|
||||||
(
|
|
||||||
"ios",
|
|
||||||
IOS_CHECK_IOCS_MODULES,
|
|
||||||
"BackupCheckerModule",
|
|
||||||
"BugReportCheckerModule",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"android",
|
|
||||||
ANDROID_CHECK_IOCS_MODULES,
|
|
||||||
"BugReportCheckerModule",
|
|
||||||
"BackupCheckerModule",
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_check_iocs_lists_the_custom_modules_it_runs(
|
|
||||||
platform, builtin_modules, listed, not_listed, caplog
|
|
||||||
):
|
|
||||||
cmd = CmdCheckIOCS(
|
|
||||||
custom_modules=[
|
|
||||||
CustomResultsModule,
|
|
||||||
BackupCheckerModule,
|
|
||||||
BugReportCheckerModule,
|
|
||||||
BackupOnlyModule,
|
|
||||||
],
|
|
||||||
platform=platform,
|
|
||||||
)
|
|
||||||
cmd.modules = builtin_modules
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.list_modules()
|
|
||||||
|
|
||||||
assert "CustomResultsModule" in caplog.text
|
|
||||||
# The module which implements check_indicators() for this platform is listed.
|
|
||||||
assert listed in caplog.text
|
|
||||||
# The one for the other platform is not, and neither is BackupOnlyModule.
|
|
||||||
assert not_listed not in caplog.text
|
|
||||||
assert "BackupOnlyModule" not in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
class ReplacementManifest(Manifest):
|
|
||||||
"""A replacement for a built-in module which does not declare check-iocs."""
|
|
||||||
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = Manifest
|
|
||||||
|
|
||||||
|
|
||||||
def test_check_iocs_uses_a_replacement_of_a_built_in_module():
|
|
||||||
# A replacement which subclasses a built-in module inherits its
|
|
||||||
# check_indicators(). check-iocs then runs it in place of that module.
|
|
||||||
cmd = CmdCheckIOCS(custom_modules=[ReplacementManifest], platform="ios")
|
|
||||||
cmd.modules = IOS_CHECK_IOCS_MODULES
|
|
||||||
|
|
||||||
available = cmd._available_modules()
|
|
||||||
|
|
||||||
assert ReplacementManifest in available
|
|
||||||
assert Manifest not in available
|
|
||||||
|
|
||||||
|
|
||||||
LOADED_MODULE = '''
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
|
|
||||||
|
|
||||||
class LoadedResultsModule(MVTModule):
|
|
||||||
"""A module loaded from a file with --load-module."""
|
|
||||||
|
|
||||||
slug = "loaded_results"
|
|
||||||
supported_commands = (("ios", "check-iocs"), ("android", "check-iocs"))
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
|
||||||
self.log.warning("loaded module checked %d results", len(self.results))
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("cli", [ios_cli, android_cli], ids=["mvt-ios", "mvt-android"])
|
|
||||||
def test_check_iocs_loads_custom_modules_from_a_file_on_each_cli(cli, tmp_path, caplog):
|
|
||||||
module_path = tmp_path / "loaded_module.py"
|
|
||||||
module_path.write_text(LOADED_MODULE)
|
|
||||||
results_folder = tmp_path / "results"
|
|
||||||
results_folder.mkdir()
|
|
||||||
(results_folder / "loaded_results.json").write_text(json.dumps([{"a": 1}]))
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
result = CliRunner().invoke(
|
|
||||||
cli,
|
|
||||||
[
|
|
||||||
*OFFLINE,
|
|
||||||
"check-iocs",
|
|
||||||
"--load-module",
|
|
||||||
str(module_path),
|
|
||||||
str(results_folder),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
assert result.exit_code == 0, result.output
|
|
||||||
assert "loaded module checked 1 results" in caplog.text
|
|
||||||
@@ -1,236 +0,0 @@
|
|||||||
import json
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from click.testing import CliRunner
|
|
||||||
|
|
||||||
from mvt.android.cli import cli as android_cli
|
|
||||||
from mvt.cli import cli as mvt_cli
|
|
||||||
from mvt.common.cli_plugins import (
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
)
|
|
||||||
from mvt.common.cmd_plugins import plugins
|
|
||||||
from mvt.common.module import MVTModule
|
|
||||||
from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP
|
|
||||||
from mvt.common.updates import PluginUpdates
|
|
||||||
from mvt.ios.cli import cli as ios_cli
|
|
||||||
|
|
||||||
|
|
||||||
class ExampleModule(MVTModule):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class AnotherModule(MVTModule):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class FakeDistribution:
|
|
||||||
def __init__(self, name, version="1.0.0", direct_url=None):
|
|
||||||
self.name = name
|
|
||||||
self.version = version
|
|
||||||
self.direct_url = direct_url
|
|
||||||
|
|
||||||
def read_text(self, file_name):
|
|
||||||
if file_name == "direct_url.json" and self.direct_url is not None:
|
|
||||||
return json.dumps(self.direct_url)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _entry_point(name, distribution, modules=None, exception=None):
|
|
||||||
def load():
|
|
||||||
if exception is not None:
|
|
||||||
raise exception
|
|
||||||
return modules
|
|
||||||
|
|
||||||
return SimpleNamespace(
|
|
||||||
name=name, value="example_plugin:modules", dist=distribution, load=load
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _run(command, arguments):
|
|
||||||
# Keep rich from wrapping the table while its content is being asserted.
|
|
||||||
return CliRunner().invoke(command, arguments, env={"COLUMNS": "200"})
|
|
||||||
|
|
||||||
|
|
||||||
def _table_rows(output):
|
|
||||||
"""Return the content of the table rows, without the header and the box."""
|
|
||||||
return [
|
|
||||||
[cell.strip() for cell in line.strip().strip("│").split("│")]
|
|
||||||
for line in output.splitlines()
|
|
||||||
if "│" in line
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _table_header(output):
|
|
||||||
for line in output.splitlines():
|
|
||||||
if "┃" in line:
|
|
||||||
return [cell.strip() for cell in line.strip().strip("┃").split("┃")]
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def _install(monkeypatch, distributions, entry_points):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cmd_plugins.installed_plugin_distributions",
|
|
||||||
lambda: distributions,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cmd_plugins.importlib.metadata.entry_points",
|
|
||||||
lambda *, group: entry_points.get(group, []),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_plugins_is_a_builtin_command_of_the_mvt_cli_only():
|
|
||||||
assert mvt_cli.commands["plugins"] is plugins
|
|
||||||
assert "plugins" not in ios_cli.commands
|
|
||||||
assert "plugins" not in android_cli.commands
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_shows_what_every_plugin_contributes(monkeypatch):
|
|
||||||
index_plugin = FakeDistribution("example-plugin", version="1.2.0")
|
|
||||||
repository_plugin = FakeDistribution(
|
|
||||||
"repository-plugin",
|
|
||||||
version="0.1.0",
|
|
||||||
direct_url={
|
|
||||||
"url": "https://example.org/plugin.git",
|
|
||||||
"vcs_info": {"vcs": "git", "commit_id": "b" * 40},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
local_plugin = FakeDistribution(
|
|
||||||
"local-plugin",
|
|
||||||
direct_url={"url": "file:///plugins", "dir_info": {"editable": True}},
|
|
||||||
)
|
|
||||||
_install(
|
|
||||||
monkeypatch,
|
|
||||||
[index_plugin, local_plugin, repository_plugin],
|
|
||||||
{
|
|
||||||
MODULES_ENTRY_POINT_GROUP: [
|
|
||||||
_entry_point(
|
|
||||||
"example", index_plugin, modules=[ExampleModule, AnotherModule]
|
|
||||||
),
|
|
||||||
_entry_point("local", local_plugin, modules=lambda: [ExampleModule]),
|
|
||||||
],
|
|
||||||
IOS_CLI_PLUGIN_GROUP: [_entry_point("summarize", repository_plugin)],
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP: [_entry_point("triage", local_plugin)],
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP: [_entry_point("report", repository_plugin)],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result = _run(plugins, ["list"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
# Plugins are listed by name, with the modules and the commands each of
|
|
||||||
# them contributes.
|
|
||||||
assert _table_header(result.output) == [
|
|
||||||
"Name",
|
|
||||||
"Version",
|
|
||||||
"Origin",
|
|
||||||
"Modules",
|
|
||||||
"Commands",
|
|
||||||
]
|
|
||||||
assert _table_rows(result.output) == [
|
|
||||||
["example-plugin", "1.2.0", "pypi", "2", "-"],
|
|
||||||
["local-plugin", "1.0.0", "local", "1", "triage"],
|
|
||||||
["repository-plugin", "0.1.0", "git+bbbbbbbb", "0", "report, summarize"],
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_reports_a_broken_module_entry_point(monkeypatch):
|
|
||||||
plugin = FakeDistribution("broken-plugin")
|
|
||||||
_install(
|
|
||||||
monkeypatch,
|
|
||||||
[plugin],
|
|
||||||
{
|
|
||||||
MODULES_ENTRY_POINT_GROUP: [
|
|
||||||
_entry_point(
|
|
||||||
"broken", plugin, exception=ImportError("missing dependency")
|
|
||||||
)
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result = _run(plugins, ["list"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert _table_rows(result.output) == [
|
|
||||||
["broken-plugin", "1.0.0", "pypi", "error", "-"]
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_without_plugins(monkeypatch):
|
|
||||||
_install(monkeypatch, [], {})
|
|
||||||
|
|
||||||
result = _run(plugins, ["list"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert result.output == "No MVT plugins are installed.\n"
|
|
||||||
|
|
||||||
|
|
||||||
def test_check_updates_prints_the_findings_and_ignores_the_throttle(monkeypatch):
|
|
||||||
findings = [
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
_install(monkeypatch, [FakeDistribution("example-plugin")], {})
|
|
||||||
monkeypatch.setattr(PluginUpdates, "check", lambda self: findings)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
PluginUpdates,
|
|
||||||
"should_check",
|
|
||||||
lambda self: pytest.fail("an explicit check must not be throttled"),
|
|
||||||
)
|
|
||||||
|
|
||||||
result = _run(plugins, ["check-updates"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert "Plugin updates available:" in result.output
|
|
||||||
assert "example-plugin 1.0.0 → 1.2.0" in result.output
|
|
||||||
assert "Upgrade with: pip install -U example-plugin" in result.output
|
|
||||||
assert "MVT does not install plugin updates." in result.output
|
|
||||||
|
|
||||||
|
|
||||||
def test_check_updates_without_available_updates(monkeypatch):
|
|
||||||
_install(monkeypatch, [FakeDistribution("example-plugin")], {})
|
|
||||||
monkeypatch.setattr(PluginUpdates, "check", lambda self: [])
|
|
||||||
|
|
||||||
result = _run(plugins, ["check-updates"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert "All plugins are up to date." in result.output
|
|
||||||
|
|
||||||
|
|
||||||
def test_check_updates_without_plugins(monkeypatch):
|
|
||||||
_install(monkeypatch, [], {})
|
|
||||||
monkeypatch.setattr(
|
|
||||||
PluginUpdates,
|
|
||||||
"check",
|
|
||||||
lambda self: pytest.fail("nothing must be checked without plugins"),
|
|
||||||
)
|
|
||||||
|
|
||||||
result = _run(plugins, ["check-updates"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert "No MVT plugins are installed." in result.output
|
|
||||||
|
|
||||||
|
|
||||||
def test_check_updates_without_network_access(monkeypatch):
|
|
||||||
monkeypatch.setattr("mvt.common.cmd_plugins.settings.NETWORK_ACCESS_ALLOWED", False)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.cmd_plugins.installed_plugin_distributions",
|
|
||||||
lambda: pytest.fail("plugins must not be listed without network access"),
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
PluginUpdates,
|
|
||||||
"check",
|
|
||||||
lambda self: pytest.fail("nothing must be checked without network access"),
|
|
||||||
)
|
|
||||||
|
|
||||||
result = _run(plugins, ["check-updates"])
|
|
||||||
|
|
||||||
assert result.exit_code == 0
|
|
||||||
assert "Network access is disabled" in result.output
|
|
||||||
@@ -68,123 +68,6 @@ class CustomDependsOnBuiltin(RecordingModule):
|
|||||||
dependencies = (FirstModule,)
|
dependencies = (FirstModule,)
|
||||||
|
|
||||||
|
|
||||||
class ReplacementModule(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
super().run()
|
|
||||||
self.results = ["replacement"]
|
|
||||||
|
|
||||||
|
|
||||||
class OtherReplacementModule(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
|
|
||||||
|
|
||||||
class UnrelatedReplacementModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = IndependentModule
|
|
||||||
|
|
||||||
|
|
||||||
class ReplacementOfReplacementModule(ReplacementModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = ReplacementModule
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
super().run()
|
|
||||||
self.results = ["replacement of replacement"]
|
|
||||||
|
|
||||||
|
|
||||||
class ReplacesOwnDependencyModule(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
dependencies = (FirstModule,)
|
|
||||||
|
|
||||||
|
|
||||||
class DisabledReplacementModule(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
enabled = False
|
|
||||||
|
|
||||||
|
|
||||||
class MutualReplacementOne(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
|
|
||||||
|
|
||||||
class MutualReplacementTwo(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = MutualReplacementOne
|
|
||||||
|
|
||||||
|
|
||||||
MutualReplacementOne.replaces = MutualReplacementTwo
|
|
||||||
|
|
||||||
|
|
||||||
class CycleOneModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
|
|
||||||
|
|
||||||
class CycleTwoModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = CycleOneModule
|
|
||||||
|
|
||||||
|
|
||||||
class CycleThreeModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = CycleTwoModule
|
|
||||||
|
|
||||||
|
|
||||||
CycleOneModule.replaces = CycleThreeModule
|
|
||||||
|
|
||||||
|
|
||||||
class UnavailableDependencyModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
|
|
||||||
|
|
||||||
class ReplacementMissingDependency(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
dependencies = (UnavailableDependencyModule,)
|
|
||||||
|
|
||||||
|
|
||||||
class SharedSlugModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
slug = "shared_slug"
|
|
||||||
|
|
||||||
|
|
||||||
class OtherSharedSlugModule(RecordingModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
slug = "shared_slug"
|
|
||||||
|
|
||||||
|
|
||||||
class ReplacementKeepingTheSlug(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
slug = "first_module"
|
|
||||||
|
|
||||||
|
|
||||||
class SameNameReplacementModule(FirstModule):
|
|
||||||
supported_commands = (("ios", "check-backup"),)
|
|
||||||
replaces = FirstModule
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
super().run()
|
|
||||||
self.results = ["replacement"]
|
|
||||||
|
|
||||||
|
|
||||||
# Modules replacing a built-in one usually keep its class name, which is the
|
|
||||||
# name `--module` matches on.
|
|
||||||
SameNameReplacementModule.__name__ = "FirstModule"
|
|
||||||
|
|
||||||
|
|
||||||
def logged_substitutions(caplog) -> list[str]:
|
|
||||||
return [
|
|
||||||
record.getMessage()
|
|
||||||
for record in caplog.records
|
|
||||||
if record.levelno == logging.INFO and "replaces module" in record.getMessage()
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
class RecordingCommand(Command):
|
class RecordingCommand(Command):
|
||||||
def init(self):
|
def init(self):
|
||||||
self.initialized = True
|
self.initialized = True
|
||||||
@@ -420,376 +303,3 @@ class TestCommand:
|
|||||||
cmd.run()
|
cmd.run()
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"]
|
assert RecordingModule.run_order == ["FirstModule", "CustomDependsOnBuiltin"]
|
||||||
|
|
||||||
def test_custom_module_replaces_builtin(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule, IndependentModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"]
|
|
||||||
assert cmd.module_replacements == {FirstModule: ReplacementModule}
|
|
||||||
assert (
|
|
||||||
"Module ReplacementModule from" in caplog.text
|
|
||||||
and "replaces module FirstModule" in caplog.text
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_modules_without_replaces_all_run(self):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [CustomIOSBackupModule]
|
|
||||||
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["FirstModule", "CustomIOSBackupModule"]
|
|
||||||
assert cmd.module_replacements == {}
|
|
||||||
|
|
||||||
def test_unavailable_replaced_module_is_ignored(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [IndependentModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["IndependentModule", "ReplacementModule"]
|
|
||||||
assert cmd.module_replacements == {}
|
|
||||||
assert "replaces module FirstModule" not in caplog.text
|
|
||||||
|
|
||||||
def test_dependencies_are_resolved_to_the_replacement(self):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [SecondModule, FirstModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule]
|
|
||||||
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["ReplacementModule", "SecondModule"]
|
|
||||||
second = next(
|
|
||||||
module for module in cmd.executed if isinstance(module, SecondModule)
|
|
||||||
)
|
|
||||||
assert isinstance(second.dependency_modules[FirstModule], ReplacementModule)
|
|
||||||
assert second.results == ["replacement", "second"]
|
|
||||||
|
|
||||||
def test_replacement_with_unavailable_dependency_skips_its_dependents(
|
|
||||||
self, caplog
|
|
||||||
):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule, SecondModule, IndependentModule]
|
|
||||||
cmd.custom_modules = [ReplacementMissingDependency]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
# The replacement cannot run, and the module it replaced was dropped
|
|
||||||
# from the run by the replacement, so neither of them produces
|
|
||||||
# results and the module depending on the replaced one is skipped.
|
|
||||||
assert RecordingModule.run_order == ["IndependentModule"]
|
|
||||||
assert (
|
|
||||||
"Module ReplacementMissingDependency will be SKIPPED: it depends "
|
|
||||||
"on module UnavailableDependencyModule, which is not available in "
|
|
||||||
"this command." in caplog.text
|
|
||||||
)
|
|
||||||
# The skipped dependent is told which module it actually depends on,
|
|
||||||
# and the module class its author declared.
|
|
||||||
assert (
|
|
||||||
"Module SecondModule will be SKIPPED: it depends on module "
|
|
||||||
"ReplacementMissingDependency (replacing module FirstModule), "
|
|
||||||
"itself skipped for depending on unavailable module "
|
|
||||||
"UnavailableDependencyModule." in caplog.text
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_selected_replacement_with_unavailable_dependency_runs_nothing(
|
|
||||||
self, caplog, tmp_path
|
|
||||||
):
|
|
||||||
cmd = RecordingCommand(module_name="FirstModule", results_path=str(tmp_path))
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule, IndependentModule]
|
|
||||||
cmd.custom_modules = [ReplacementMissingDependency]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == []
|
|
||||||
assert not hasattr(cmd, "initialized")
|
|
||||||
assert (
|
|
||||||
"Module FirstModule was replaced by module "
|
|
||||||
"ReplacementMissingDependency, which is run in its place."
|
|
||||||
in caplog.text
|
|
||||||
)
|
|
||||||
assert "Module ReplacementMissingDependency will be SKIPPED" in caplog.text
|
|
||||||
assert "No modules will be run" in caplog.text
|
|
||||||
# Nothing else was selected, so the warnings must not promise that the
|
|
||||||
# analysis continues right before saying that it does not.
|
|
||||||
assert "The rest of the analysis will still run" not in caplog.text
|
|
||||||
# No module ran, so no results were stored next to the command log.
|
|
||||||
assert [path.name for path in tmp_path.iterdir()] == ["command.log"]
|
|
||||||
|
|
||||||
def test_chained_replacements_are_resolved(self):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [SecondModule, FirstModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule, ReplacementOfReplacementModule]
|
|
||||||
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == [
|
|
||||||
"ReplacementOfReplacementModule",
|
|
||||||
"SecondModule",
|
|
||||||
]
|
|
||||||
second = next(
|
|
||||||
module for module in cmd.executed if isinstance(module, SecondModule)
|
|
||||||
)
|
|
||||||
assert second.results == ["replacement of replacement", "second"]
|
|
||||||
|
|
||||||
def test_replacement_which_is_not_a_subclass_warns(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [IndependentModule]
|
|
||||||
cmd.custom_modules = [UnrelatedReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["UnrelatedReplacementModule"]
|
|
||||||
assert "is not a subclass of it" in caplog.text
|
|
||||||
|
|
||||||
def test_multiple_modules_replacing_the_same_module_warn(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule, OtherReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == [
|
|
||||||
"ReplacementModule",
|
|
||||||
"OtherReplacementModule",
|
|
||||||
]
|
|
||||||
assert cmd.module_replacements == {FirstModule: ReplacementModule}
|
|
||||||
assert (
|
|
||||||
"Modules ReplacementModule and OtherReplacementModule both replace "
|
|
||||||
"module FirstModule. Both of them will run, FirstModule will not, "
|
|
||||||
"and modules depending on FirstModule will use the results of "
|
|
||||||
"ReplacementModule." in caplog.text
|
|
||||||
)
|
|
||||||
assert "overwrite each other's results file" in caplog.text
|
|
||||||
|
|
||||||
def test_module_replacing_its_own_dependency_runs(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [ReplacesOwnDependencyModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["ReplacesOwnDependencyModule"]
|
|
||||||
assert (
|
|
||||||
"Module ReplacesOwnDependencyModule depends on module FirstModule, "
|
|
||||||
"which it also replaces" in caplog.text
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_literal_self_dependency_is_still_circular(self, caplog):
|
|
||||||
class SelfDependentModule(RecordingModule):
|
|
||||||
pass
|
|
||||||
|
|
||||||
SelfDependentModule.dependencies = (SelfDependentModule,)
|
|
||||||
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.modules = [SelfDependentModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == []
|
|
||||||
assert not hasattr(cmd, "initialized")
|
|
||||||
assert "Circular module dependency detected" in caplog.text
|
|
||||||
|
|
||||||
def test_disabled_replacement_keeps_the_replaced_module(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [DisabledReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["FirstModule"]
|
|
||||||
assert cmd.module_replacements == {}
|
|
||||||
assert logged_substitutions(caplog) == []
|
|
||||||
|
|
||||||
def test_modules_replacing_each_other_all_run(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.custom_modules = [MutualReplacementOne, MutualReplacementTwo]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == [
|
|
||||||
"MutualReplacementOne",
|
|
||||||
"MutualReplacementTwo",
|
|
||||||
]
|
|
||||||
assert cmd.module_replacements == {}
|
|
||||||
assert (
|
|
||||||
"Modules MutualReplacementOne, MutualReplacementTwo replace each "
|
|
||||||
"other in a cycle" in caplog.text
|
|
||||||
)
|
|
||||||
assert logged_substitutions(caplog) == []
|
|
||||||
|
|
||||||
def test_replacement_cycle_of_three_modules_all_run(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.custom_modules = [CycleOneModule, CycleTwoModule, CycleThreeModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == [
|
|
||||||
"CycleOneModule",
|
|
||||||
"CycleTwoModule",
|
|
||||||
"CycleThreeModule",
|
|
||||||
]
|
|
||||||
assert cmd.module_replacements == {}
|
|
||||||
assert (
|
|
||||||
"Modules CycleOneModule, CycleThreeModule, CycleTwoModule replace "
|
|
||||||
"each other in a cycle" in caplog.text
|
|
||||||
)
|
|
||||||
assert logged_substitutions(caplog) == []
|
|
||||||
|
|
||||||
def test_selected_replaced_module_name_runs_the_replacement(self, caplog):
|
|
||||||
cmd = RecordingCommand(module_name="FirstModule")
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == ["ReplacementModule"]
|
|
||||||
assert (
|
|
||||||
"Module FirstModule was replaced by module ReplacementModule"
|
|
||||||
in caplog.text
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_unknown_selected_module_warns_and_stops(self, caplog):
|
|
||||||
cmd = RecordingCommand(module_name="NoSuchModule")
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == []
|
|
||||||
assert not hasattr(cmd, "initialized")
|
|
||||||
assert (
|
|
||||||
"No module named NoSuchModule is available for the check-backup "
|
|
||||||
"command" in caplog.text
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_selected_module_name_matches_the_replacement(self):
|
|
||||||
cmd = RecordingCommand(module_name="FirstModule")
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [SameNameReplacementModule]
|
|
||||||
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert len(cmd.executed) == 1
|
|
||||||
assert isinstance(cmd.executed[0], SameNameReplacementModule)
|
|
||||||
assert cmd.executed[0].results == ["replacement"]
|
|
||||||
|
|
||||||
def test_list_modules_reflects_replacements(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule, IndependentModule]
|
|
||||||
cmd.custom_modules = [ReplacementModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO):
|
|
||||||
cmd.list_modules()
|
|
||||||
|
|
||||||
listed = [
|
|
||||||
record.getMessage()
|
|
||||||
for record in caplog.records
|
|
||||||
if "Modules from" in record.getMessage()
|
|
||||||
]
|
|
||||||
assert any("ReplacementModule" in message for message in listed)
|
|
||||||
assert not any("FirstModule" in message for message in listed)
|
|
||||||
|
|
||||||
def test_modules_sharing_a_slug_are_reported(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.custom_modules = [SharedSlugModule, OtherSharedSlugModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert RecordingModule.run_order == [
|
|
||||||
"SharedSlugModule",
|
|
||||||
"OtherSharedSlugModule",
|
|
||||||
]
|
|
||||||
collisions = [
|
|
||||||
record.getMessage()
|
|
||||||
for record in caplog.records
|
|
||||||
if "both use the slug" in record.getMessage()
|
|
||||||
]
|
|
||||||
assert len(collisions) == 1
|
|
||||||
assert "Modules SharedSlugModule from" in collisions[0]
|
|
||||||
assert "and OtherSharedSlugModule from" in collisions[0]
|
|
||||||
assert "both use the slug shared_slug" in collisions[0]
|
|
||||||
assert "overwrites the results of the other in shared_slug.json" in (
|
|
||||||
collisions[0]
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_replacement_keeping_the_replaced_slug_is_not_reported(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule]
|
|
||||||
cmd.custom_modules = [ReplacementKeepingTheSlug]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
# The replaced module is no longer part of the run, so taking over its
|
|
||||||
# slug is what the replacement is for, not a collision.
|
|
||||||
assert RecordingModule.run_order == ["ReplacementKeepingTheSlug"]
|
|
||||||
assert ReplacementKeepingTheSlug.get_slug() == FirstModule.get_slug()
|
|
||||||
assert "both use the slug" not in caplog.text
|
|
||||||
|
|
||||||
def test_modules_with_distinct_slugs_are_not_reported(self, caplog):
|
|
||||||
cmd = RecordingCommand()
|
|
||||||
cmd.platform = "ios"
|
|
||||||
cmd.name = "check-backup"
|
|
||||||
cmd.modules = [FirstModule, IndependentModule]
|
|
||||||
cmd.custom_modules = [CustomIOSBackupModule]
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
cmd.run()
|
|
||||||
|
|
||||||
assert "both use the slug" not in caplog.text
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
from mvt.android.command_modules import ANDROID_CHECK_IOCS_MODULES
|
|
||||||
from mvt.android.modules.androidqf import ANDROIDQF_MODULES
|
|
||||||
from mvt.android.modules.backup import BACKUP_MODULES as ANDROID_BACKUP_MODULES
|
|
||||||
from mvt.android.modules.bugreport import BUGREPORT_MODULES
|
|
||||||
from mvt.android.modules.intrusion_logs import INTRUSION_LOGS_MODULES
|
|
||||||
from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES
|
|
||||||
from mvt.ios.modules.backup import BACKUP_MODULES as IOS_BACKUP_MODULES
|
|
||||||
from mvt.ios.modules.fs import FS_MODULES
|
|
||||||
from mvt.ios.modules.mixed import MIXED_MODULES
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_check_iocs_lists_are_the_families_of_their_platform():
|
|
||||||
# The CLI reads these same lists, so nothing composing one elsewhere can
|
|
||||||
# drift from what the command runs. This pins what the lists are composed
|
|
||||||
# of.
|
|
||||||
assert IOS_CHECK_IOCS_MODULES == IOS_BACKUP_MODULES + FS_MODULES + MIXED_MODULES
|
|
||||||
assert ANDROID_CHECK_IOCS_MODULES == (
|
|
||||||
ANDROID_BACKUP_MODULES
|
|
||||||
+ BUGREPORT_MODULES
|
|
||||||
+ ANDROIDQF_MODULES
|
|
||||||
+ INTRUSION_LOGS_MODULES
|
|
||||||
)
|
|
||||||
@@ -1,14 +1,9 @@
|
|||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from mvt.common.cli_plugins import _module_name_for_path as _command_module_name
|
|
||||||
from mvt.common.module import MVTModule
|
from mvt.common.module import MVTModule
|
||||||
from mvt.common.module_loader import (
|
from mvt.common.module_loader import (
|
||||||
CustomModuleLoadError,
|
CustomModuleLoadError,
|
||||||
_module_name_for_path,
|
|
||||||
get_module_logger,
|
get_module_logger,
|
||||||
get_plugin_logger,
|
|
||||||
load_custom_modules,
|
load_custom_modules,
|
||||||
load_custom_modules_from_path,
|
load_custom_modules_from_path,
|
||||||
module_supports_command,
|
module_supports_command,
|
||||||
@@ -173,9 +168,9 @@ def test_get_module_logger_strips_the_plugin_package_prefix():
|
|||||||
class PluginModule(MVTModule):
|
class PluginModule(MVTModule):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
PluginModule.__module__ = "mvt_plugin_example_org.ios.custom"
|
PluginModule.__module__ = "mvt_plugin_amnesty_custom.ios.custom"
|
||||||
|
|
||||||
assert get_module_logger(PluginModule).name == "mvt.ext.example_org.ios.custom"
|
assert get_module_logger(PluginModule).name == "mvt.ext.amnesty_custom.ios.custom"
|
||||||
|
|
||||||
|
|
||||||
def test_get_module_logger_only_strips_the_prefix_from_the_top_level():
|
def test_get_module_logger_only_strips_the_prefix_from_the_top_level():
|
||||||
@@ -192,25 +187,3 @@ def test_get_module_logger_names_path_modules_after_their_file(tmp_path):
|
|||||||
module = load_custom_modules_from_path(str(module_path))[0]
|
module = load_custom_modules_from_path(str(module_path))[0]
|
||||||
|
|
||||||
assert get_module_logger(module).name == "mvt.ext.my_custom_module"
|
assert get_module_logger(module).name == "mvt.ext.my_custom_module"
|
||||||
|
|
||||||
|
|
||||||
def test_get_plugin_logger_uses_the_same_namespace_as_modules():
|
|
||||||
assert (
|
|
||||||
get_plugin_logger("mvt_plugin_example_org.commands.summarize").name
|
|
||||||
== "mvt.ext.example_org.commands.summarize"
|
|
||||||
)
|
|
||||||
assert get_plugin_logger("example_plugin.cli").name == "mvt.ext.example_plugin.cli"
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_plugin_logger_keeps_builtin_names():
|
|
||||||
assert get_plugin_logger("mvt.ios.cli").name == "mvt.ios.cli"
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_plugin_logger_names_loaded_files_after_the_file():
|
|
||||||
# A file loaded with --load-command or --load-module is imported under a
|
|
||||||
# mangled name. The log names the file instead.
|
|
||||||
command_name = _command_module_name(Path("/tmp/case_summary.py"))
|
|
||||||
module_name = _module_name_for_path(Path("/tmp/my_custom_module.py"))
|
|
||||||
|
|
||||||
assert get_plugin_logger(command_name).name == "mvt.ext.case_summary"
|
|
||||||
assert get_plugin_logger(module_name).name == "mvt.ext.my_custom_module"
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# Mobile Verification Toolkit (MVT)
|
|
||||||
# Copyright (c) 2021-2026 The MVT Authors.
|
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
|
||||||
# https://license.mvt.re/1.1/
|
|
||||||
|
|
||||||
import mvt.plugin
|
|
||||||
from mvt.android.modules.backup.base import BackupModule
|
|
||||||
from mvt.common.config import settings
|
|
||||||
|
|
||||||
from ..plugin_fixtures import run_isolated_python
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_exported_names_are_the_public_names():
|
|
||||||
public = {name for name in vars(mvt.plugin) if not name.startswith("_")}
|
|
||||||
|
|
||||||
assert public == set(mvt.plugin.__all__)
|
|
||||||
assert mvt.plugin.settings is settings
|
|
||||||
assert mvt.plugin.AndroidBackupModule is BackupModule
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_surface_imports_before_anything_else_of_mvt(tmp_path):
|
|
||||||
# A plugin can import the surface as its first import of MVT. The
|
|
||||||
# subprocess gets a temporary home because importing MVT writes its
|
|
||||||
# configuration file.
|
|
||||||
result = run_isolated_python(
|
|
||||||
"from mvt.plugin import IOSExtraction, MVT_VERSION, settings\n"
|
|
||||||
"assert MVT_VERSION\n"
|
|
||||||
"assert settings.NETWORK_TIMEOUT > 0\n"
|
|
||||||
"assert IOSExtraction.__name__ == 'IOSExtraction'\n",
|
|
||||||
home=tmp_path,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert result.returncode == 0, result.stderr
|
|
||||||
assert result.stderr == ""
|
|
||||||
@@ -1,711 +0,0 @@
|
|||||||
import json
|
|
||||||
import shlex
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from mvt.common import logo
|
|
||||||
from mvt.common.cli_plugins import (
|
|
||||||
ANDROID_CLI_PLUGIN_GROUP,
|
|
||||||
IOS_CLI_PLUGIN_GROUP,
|
|
||||||
NEUTRAL_CLI_PLUGIN_GROUP,
|
|
||||||
)
|
|
||||||
from mvt.common.module_loader import MODULES_ENTRY_POINT_GROUP
|
|
||||||
from mvt.common.updates import (
|
|
||||||
MVTUpdates,
|
|
||||||
PluginUpdates,
|
|
||||||
installed_plugin_distributions,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
REPOSITORY_URL = "https://example.org/plugin.git"
|
|
||||||
INSTALLED_COMMIT = "a" * 40
|
|
||||||
REMOTE_COMMIT = "b" * 40
|
|
||||||
|
|
||||||
|
|
||||||
class FakeDistribution:
|
|
||||||
def __init__(self, name, version="1.0.0", direct_url=None):
|
|
||||||
self.name = name
|
|
||||||
self.version = version
|
|
||||||
self.direct_url = direct_url
|
|
||||||
|
|
||||||
def read_text(self, file_name):
|
|
||||||
if file_name == "direct_url.json" and self.direct_url is not None:
|
|
||||||
return json.dumps(self.direct_url)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
class FakeResponse:
|
|
||||||
def __init__(self, status_code=200, payload=None):
|
|
||||||
self.status_code = status_code
|
|
||||||
self.payload = payload or {}
|
|
||||||
|
|
||||||
def json(self):
|
|
||||||
return self.payload
|
|
||||||
|
|
||||||
|
|
||||||
def _entry_point(name, distribution, value="plugin:modules"):
|
|
||||||
return SimpleNamespace(name=name, value=value, dist=distribution)
|
|
||||||
|
|
||||||
|
|
||||||
def _git_distribution(requested_revision=None, commit=INSTALLED_COMMIT):
|
|
||||||
vcs_info = {"vcs": "git", "commit_id": commit}
|
|
||||||
if requested_revision:
|
|
||||||
vcs_info["requested_revision"] = requested_revision
|
|
||||||
|
|
||||||
return FakeDistribution(
|
|
||||||
"example-plugin",
|
|
||||||
direct_url={"url": REPOSITORY_URL, "vcs_info": vcs_info},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _fake_git(stdout="", returncode=0, exception=None, calls=None):
|
|
||||||
def run(command, **kwargs):
|
|
||||||
if calls is not None:
|
|
||||||
calls.append((command, kwargs))
|
|
||||||
if exception is not None:
|
|
||||||
raise exception
|
|
||||||
return SimpleNamespace(returncode=returncode, stdout=stdout, stderr="")
|
|
||||||
|
|
||||||
return run
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def data_folder(tmp_path, monkeypatch):
|
|
||||||
folder = tmp_path / "mvt-data"
|
|
||||||
monkeypatch.setattr("mvt.common.updates.MVT_DATA_FOLDER", str(folder))
|
|
||||||
return folder
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def one_plugin(monkeypatch):
|
|
||||||
def install(distribution):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.installed_plugin_distributions",
|
|
||||||
lambda: [distribution],
|
|
||||||
)
|
|
||||||
return distribution
|
|
||||||
|
|
||||||
return install
|
|
||||||
|
|
||||||
|
|
||||||
def test_installed_plugin_distributions_covers_every_plugin_group(monkeypatch):
|
|
||||||
zeta = FakeDistribution("zeta-plugin")
|
|
||||||
alpha = FakeDistribution("alpha-plugin")
|
|
||||||
neutral = FakeDistribution("neutral-plugin")
|
|
||||||
|
|
||||||
def entry_points(*, group):
|
|
||||||
if group == MODULES_ENTRY_POINT_GROUP:
|
|
||||||
return [_entry_point("zeta", zeta), _entry_point("alpha", alpha)]
|
|
||||||
if group == IOS_CLI_PLUGIN_GROUP:
|
|
||||||
return [_entry_point("zeta-ios", zeta)]
|
|
||||||
if group == ANDROID_CLI_PLUGIN_GROUP:
|
|
||||||
return [_entry_point("alpha-android", alpha)]
|
|
||||||
if group == NEUTRAL_CLI_PLUGIN_GROUP:
|
|
||||||
return [_entry_point("shared", neutral)]
|
|
||||||
return []
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.importlib.metadata.entry_points", entry_points
|
|
||||||
)
|
|
||||||
|
|
||||||
distributions = installed_plugin_distributions()
|
|
||||||
|
|
||||||
assert [distribution.name for distribution in distributions] == [
|
|
||||||
"alpha-plugin",
|
|
||||||
"neutral-plugin",
|
|
||||||
"zeta-plugin",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_installed_plugin_distributions_skips_mvt_and_orphan_entry_points(monkeypatch):
|
|
||||||
entry_points = [
|
|
||||||
_entry_point("builtin", FakeDistribution("mvt")),
|
|
||||||
SimpleNamespace(name="orphan", value="plugin:modules", dist=None),
|
|
||||||
_entry_point("plugin", FakeDistribution("example-plugin")),
|
|
||||||
]
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.importlib.metadata.entry_points",
|
|
||||||
lambda **kwargs: entry_points,
|
|
||||||
)
|
|
||||||
|
|
||||||
distributions = installed_plugin_distributions()
|
|
||||||
|
|
||||||
assert [distribution.name for distribution in distributions] == ["example-plugin"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_installed_plugin_distributions_survives_broken_metadata(monkeypatch, caplog):
|
|
||||||
def entry_points(*, group):
|
|
||||||
raise RuntimeError("invalid package metadata")
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.importlib.metadata.entry_points", entry_points
|
|
||||||
)
|
|
||||||
|
|
||||||
assert installed_plugin_distributions() == []
|
|
||||||
assert "Unable to discover installed plugin packages" in caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
def test_index_plugin_update_is_reported(monkeypatch, data_folder, one_plugin):
|
|
||||||
one_plugin(FakeDistribution("example-plugin", version="1.0.0"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.requests.get",
|
|
||||||
lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}),
|
|
||||||
)
|
|
||||||
|
|
||||||
findings = PluginUpdates().check()
|
|
||||||
|
|
||||||
assert findings == [
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_index_plugin_queries_the_package_index_with_the_configured_timeout(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(FakeDistribution("example-plugin"))
|
|
||||||
requests_made = []
|
|
||||||
|
|
||||||
def get(url, **kwargs):
|
|
||||||
requests_made.append((url, kwargs))
|
|
||||||
return FakeResponse(payload={"info": {"version": "1.0.0"}})
|
|
||||||
|
|
||||||
monkeypatch.setattr("mvt.common.updates.requests.get", get)
|
|
||||||
monkeypatch.setattr("mvt.common.updates.settings.NETWORK_TIMEOUT", 3)
|
|
||||||
|
|
||||||
PluginUpdates().check()
|
|
||||||
|
|
||||||
assert requests_made == [
|
|
||||||
("https://pypi.org/pypi/example-plugin/json", {"timeout": 3})
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_up_to_date_index_plugin_is_not_reported(monkeypatch, data_folder, one_plugin):
|
|
||||||
one_plugin(FakeDistribution("example-plugin", version="1.2.0"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.requests.get",
|
|
||||||
lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_unpublished_plugin_is_skipped_silently(monkeypatch, data_folder, one_plugin):
|
|
||||||
one_plugin(FakeDistribution("private-plugin"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.requests.get",
|
|
||||||
lambda url, **kwargs: FakeResponse(status_code=404),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_following_a_branch_is_reported(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision="main"))
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls),
|
|
||||||
)
|
|
||||||
monkeypatch.delenv("GIT_SSH_COMMAND", raising=False)
|
|
||||||
|
|
||||||
findings = PluginUpdates().check()
|
|
||||||
|
|
||||||
assert findings == [
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "aaaaaaaa",
|
|
||||||
"latest": "bbbbbbbb",
|
|
||||||
"origin": "git",
|
|
||||||
"upgrade_command": (
|
|
||||||
f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@main'"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
]
|
|
||||||
command, options = calls[0]
|
|
||||||
assert command == ["git", "ls-remote", REPOSITORY_URL, "main"]
|
|
||||||
assert options["env"]["GIT_TERMINAL_PROMPT"] == "0"
|
|
||||||
# ssh asks the terminal for a passphrase or a host key unless it is told
|
|
||||||
# not to, which git itself cannot prevent.
|
|
||||||
assert options["env"]["GIT_SSH_COMMAND"] == (
|
|
||||||
"ssh -o BatchMode=yes -o ConnectTimeout=10"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_batch_mode_options_come_before_the_configured_ssh_options(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision="main"))
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/main\n", calls=calls),
|
|
||||||
)
|
|
||||||
monkeypatch.setenv("GIT_SSH_COMMAND", "ssh -o BatchMode=no -i /home/analyst/key")
|
|
||||||
|
|
||||||
PluginUpdates().check()
|
|
||||||
|
|
||||||
# ssh uses the first value it is given for a keyword, so an analyst asking
|
|
||||||
# for prompts cannot bring them back, while their other options still
|
|
||||||
# apply.
|
|
||||||
assert calls[0][1]["env"]["GIT_SSH_COMMAND"] == (
|
|
||||||
"ssh -o BatchMode=yes -o ConnectTimeout=10 -o BatchMode=no -i /home/analyst/key"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_without_a_revision_follows_the_default_branch(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution())
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout=f"{REMOTE_COMMIT}\tHEAD\n", calls=calls),
|
|
||||||
)
|
|
||||||
|
|
||||||
findings = PluginUpdates().check()
|
|
||||||
|
|
||||||
assert calls[0][0] == ["git", "ls-remote", REPOSITORY_URL, "HEAD"]
|
|
||||||
assert findings[0]["upgrade_command"] == (
|
|
||||||
f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_hostile_revision_cannot_inject_into_the_upgrade_command(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
revision = "main$(id)`id`;id"
|
|
||||||
one_plugin(_git_distribution(requested_revision=revision))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/heads/{revision}\n"),
|
|
||||||
)
|
|
||||||
|
|
||||||
upgrade_command = PluginUpdates().check()[0]["upgrade_command"]
|
|
||||||
|
|
||||||
# Single quotes are the only quoting a shell does not expand anything in.
|
|
||||||
assert upgrade_command == (
|
|
||||||
f"pip install -U 'example-plugin @ git+{REPOSITORY_URL}@{revision}'"
|
|
||||||
)
|
|
||||||
assert shlex.split(upgrade_command) == [
|
|
||||||
"pip",
|
|
||||||
"install",
|
|
||||||
"-U",
|
|
||||||
f"example-plugin @ git+{REPOSITORY_URL}@{revision}",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_with_an_option_like_url_is_skipped(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(
|
|
||||||
FakeDistribution(
|
|
||||||
"example-plugin",
|
|
||||||
direct_url={
|
|
||||||
"url": "--upload-pack=touch /tmp/mvt",
|
|
||||||
"vcs_info": {"vcs": "git", "commit_id": INSTALLED_COMMIT},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
assert calls == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_with_an_option_like_revision_is_skipped(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision="--upload-pack=touch /tmp/mvt"))
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
assert calls == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_at_the_latest_commit_is_not_reported(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision="main"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout=f"{INSTALLED_COMMIT}\trefs/heads/main\n"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_commit_pinned_repository_plugin_is_never_outdated(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT))
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
assert calls == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_short_commit_pinned_repository_plugin_is_never_outdated(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision=INSTALLED_COMMIT[:10]))
|
|
||||||
calls = []
|
|
||||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", _fake_git(calls=calls))
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
assert calls == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_tag_pinned_repository_plugin_is_never_outdated(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision="v1.0.0"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout=f"{REMOTE_COMMIT}\trefs/tags/v1.0.0\n"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_is_skipped_without_git(monkeypatch, data_folder, one_plugin):
|
|
||||||
one_plugin(_git_distribution(requested_revision="main"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(exception=FileNotFoundError("git")),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_plugin_is_skipped_when_git_fails(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(_git_distribution(requested_revision="main"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.subprocess.run",
|
|
||||||
_fake_git(stdout="", returncode=128),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_local_plugin_install_is_skipped(monkeypatch, data_folder, one_plugin):
|
|
||||||
one_plugin(
|
|
||||||
FakeDistribution(
|
|
||||||
"example-plugin",
|
|
||||||
direct_url={
|
|
||||||
"url": "file:///home/analyst/example-plugin",
|
|
||||||
"dir_info": {"editable": True},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def fail(*args, **kwargs):
|
|
||||||
raise AssertionError("a local plugin install must not be checked")
|
|
||||||
|
|
||||||
monkeypatch.setattr("mvt.common.updates.requests.get", fail)
|
|
||||||
monkeypatch.setattr("mvt.common.updates.subprocess.run", fail)
|
|
||||||
|
|
||||||
assert PluginUpdates().check() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_check_stores_the_findings_and_the_check_timestamp(
|
|
||||||
monkeypatch, data_folder, one_plugin
|
|
||||||
):
|
|
||||||
one_plugin(FakeDistribution("example-plugin", version="1.0.0"))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.requests.get",
|
|
||||||
lambda url, **kwargs: FakeResponse(payload={"info": {"version": "1.2.0"}}),
|
|
||||||
)
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
|
|
||||||
findings = plugin_updates.check()
|
|
||||||
|
|
||||||
assert json.loads((data_folder / "plugin_updates.json").read_text()) == findings
|
|
||||||
assert (data_folder / "latest_plugins_check").read_text().isdigit()
|
|
||||||
assert PluginUpdates().get_findings() == findings
|
|
||||||
|
|
||||||
|
|
||||||
def test_findings_are_empty_before_the_first_check(data_folder):
|
|
||||||
assert PluginUpdates().get_findings() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_malformed_cached_findings_are_dropped(data_folder):
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
usable = {
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
}
|
|
||||||
data_folder.mkdir(parents=True, exist_ok=True)
|
|
||||||
(data_folder / "plugin_updates.json").write_text(
|
|
||||||
json.dumps(
|
|
||||||
[
|
|
||||||
{"oops": 1},
|
|
||||||
"not a finding",
|
|
||||||
{"name": "half-plugin", "installed": "1.0.0"},
|
|
||||||
{**usable, "latest": None},
|
|
||||||
usable,
|
|
||||||
]
|
|
||||||
),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert plugin_updates.get_findings() == [usable]
|
|
||||||
|
|
||||||
|
|
||||||
def test_corrupt_cached_findings_are_ignored(data_folder):
|
|
||||||
data_folder.mkdir(parents=True, exist_ok=True)
|
|
||||||
(data_folder / "plugin_updates.json").write_text("{ not json", encoding="utf-8")
|
|
||||||
|
|
||||||
assert PluginUpdates().get_findings() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_cached_findings_of_upgraded_and_removed_plugins_are_dropped(
|
|
||||||
monkeypatch, data_folder
|
|
||||||
):
|
|
||||||
findings = [
|
|
||||||
{
|
|
||||||
"name": "upgraded-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U upgraded-plugin",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "removed-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U removed-plugin",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
plugin_updates.set_findings(findings)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"mvt.common.updates.installed_plugin_distributions",
|
|
||||||
lambda: [
|
|
||||||
# The analyst upgraded this plugin since the latest check.
|
|
||||||
FakeDistribution("upgraded-plugin", version="1.2.0"),
|
|
||||||
FakeDistribution("example-plugin", version="1.0.0"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
assert plugin_updates.current_findings() == [findings[2]]
|
|
||||||
|
|
||||||
|
|
||||||
def test_cached_findings_of_updated_repository_plugins_are_dropped(data_folder):
|
|
||||||
findings = [
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "aaaaaaaa",
|
|
||||||
"latest": "bbbbbbbb",
|
|
||||||
"origin": "git",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
}
|
|
||||||
]
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
plugin_updates.set_findings(findings)
|
|
||||||
|
|
||||||
assert plugin_updates.current_findings([_git_distribution()]) == findings
|
|
||||||
assert (
|
|
||||||
plugin_updates.current_findings([_git_distribution(commit=REMOTE_COMMIT)]) == []
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_corrupt_check_timestamp_does_not_raise(data_folder):
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
data_folder.mkdir(parents=True, exist_ok=True)
|
|
||||||
(data_folder / "latest_plugins_check").write_text("truncated", encoding="utf-8")
|
|
||||||
|
|
||||||
assert plugin_updates.get_latest_check() == 0
|
|
||||||
assert plugin_updates.should_check() == (True, 0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_check_is_throttled_for_twelve_hours(data_folder):
|
|
||||||
plugin_updates = PluginUpdates()
|
|
||||||
plugin_updates.set_findings([])
|
|
||||||
|
|
||||||
recent = datetime.now() - timedelta(hours=4)
|
|
||||||
with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle:
|
|
||||||
handle.write(str(int(recent.timestamp())))
|
|
||||||
|
|
||||||
should_check, hours = plugin_updates.should_check()
|
|
||||||
assert not should_check
|
|
||||||
assert hours == 8
|
|
||||||
|
|
||||||
old = datetime.now() - timedelta(hours=13)
|
|
||||||
with open(plugin_updates.latest_check_path, "w", encoding="utf-8") as handle:
|
|
||||||
handle.write(str(int(old.timestamp())))
|
|
||||||
|
|
||||||
assert plugin_updates.should_check() == (True, 0)
|
|
||||||
|
|
||||||
|
|
||||||
def test_should_check_without_a_previous_check(data_folder):
|
|
||||||
assert PluginUpdates().should_check() == (True, 0)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def no_version_check(monkeypatch):
|
|
||||||
monkeypatch.setattr(MVTUpdates, "check", lambda self: "")
|
|
||||||
# Keep rich from wrapping the plugin lines while they are being asserted.
|
|
||||||
monkeypatch.setenv("COLUMNS", "200")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def throttled_cache(monkeypatch, data_folder):
|
|
||||||
"""Fill the findings cache and put the check inside its throttle window."""
|
|
||||||
|
|
||||||
def fill(findings, distributions):
|
|
||||||
PluginUpdates().set_findings(findings)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
logo, "installed_plugin_distributions", lambda: distributions
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
PluginUpdates,
|
|
||||||
"check",
|
|
||||||
lambda self: pytest.fail("the check must be throttled"),
|
|
||||||
)
|
|
||||||
|
|
||||||
return fill
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_prints_the_cached_plugin_updates(
|
|
||||||
capsys, no_version_check, throttled_cache
|
|
||||||
):
|
|
||||||
throttled_cache(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
[FakeDistribution("example-plugin", version="1.0.0")],
|
|
||||||
)
|
|
||||||
|
|
||||||
logo.check_updates(disable_indicator_check=True)
|
|
||||||
|
|
||||||
output = capsys.readouterr().out
|
|
||||||
assert "Plugin updates available:" in output
|
|
||||||
assert "example-plugin 1.0.0 → 1.2.0 (pip install -U example-plugin)" in output
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_does_not_print_a_cached_update_of_an_upgraded_plugin(
|
|
||||||
capsys, no_version_check, throttled_cache
|
|
||||||
):
|
|
||||||
throttled_cache(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"name": "example-plugin",
|
|
||||||
"installed": "1.0.0",
|
|
||||||
"latest": "1.2.0",
|
|
||||||
"origin": "pypi",
|
|
||||||
"upgrade_command": "pip install -U example-plugin",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
# The analyst already upgraded the plugin the cached finding is about.
|
|
||||||
[FakeDistribution("example-plugin", version="1.2.0")],
|
|
||||||
)
|
|
||||||
|
|
||||||
logo.check_updates(disable_indicator_check=True)
|
|
||||||
|
|
||||||
assert "Plugin updates" not in capsys.readouterr().out
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_prints_nothing_when_throttled_without_findings(
|
|
||||||
capsys, no_version_check, throttled_cache
|
|
||||||
):
|
|
||||||
throttled_cache([], [FakeDistribution("example-plugin")])
|
|
||||||
|
|
||||||
logo.check_updates(disable_indicator_check=True)
|
|
||||||
|
|
||||||
assert "Plugin updates" not in capsys.readouterr().out
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_survives_a_corrupt_plugin_cache(
|
|
||||||
monkeypatch, capsys, data_folder, no_version_check
|
|
||||||
):
|
|
||||||
data_folder.mkdir(parents=True, exist_ok=True)
|
|
||||||
(data_folder / "plugin_updates.json").write_text(
|
|
||||||
json.dumps([{"oops": 1}]), encoding="utf-8"
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
logo,
|
|
||||||
"installed_plugin_distributions",
|
|
||||||
lambda: [FakeDistribution("example-plugin")],
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(PluginUpdates, "should_check", lambda self: (False, 8))
|
|
||||||
|
|
||||||
logo.check_updates(disable_indicator_check=True)
|
|
||||||
|
|
||||||
assert "Plugin updates" not in capsys.readouterr().out
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_skips_the_plugin_check_without_plugins(
|
|
||||||
monkeypatch, capsys, no_version_check
|
|
||||||
):
|
|
||||||
monkeypatch.setattr(logo, "installed_plugin_distributions", list)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
PluginUpdates,
|
|
||||||
"should_check",
|
|
||||||
lambda self: pytest.fail("plugins must not be checked without plugins"),
|
|
||||||
)
|
|
||||||
|
|
||||||
logo.check_updates(disable_indicator_check=True)
|
|
||||||
|
|
||||||
assert "Plugin updates" not in capsys.readouterr().out
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_skips_the_plugin_check_without_network_access(
|
|
||||||
monkeypatch, capsys, no_version_check
|
|
||||||
):
|
|
||||||
monkeypatch.setattr("mvt.common.logo.settings.NETWORK_ACCESS_ALLOWED", False)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
logo,
|
|
||||||
"installed_plugin_distributions",
|
|
||||||
lambda: pytest.fail("plugins must not be listed without network access"),
|
|
||||||
)
|
|
||||||
|
|
||||||
logo.check_updates(disable_indicator_check=True)
|
|
||||||
|
|
||||||
assert "Plugin updates" not in capsys.readouterr().out
|
|
||||||
|
|
||||||
|
|
||||||
def test_logo_skips_the_plugin_check_when_update_checks_are_disabled(
|
|
||||||
monkeypatch, capsys
|
|
||||||
):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
logo,
|
|
||||||
"installed_plugin_distributions",
|
|
||||||
lambda: pytest.fail("plugins must not be checked with --disable-update-check"),
|
|
||||||
)
|
|
||||||
|
|
||||||
logo.check_updates(disable_version_check=True, disable_indicator_check=True)
|
|
||||||
|
|
||||||
assert capsys.readouterr().out == ""
|
|
||||||
@@ -18,7 +18,6 @@ from mvt.common.utils import (
|
|||||||
generate_hashes_from_path,
|
generate_hashes_from_path,
|
||||||
get_sha256_from_file_path,
|
get_sha256_from_file_path,
|
||||||
init_logging,
|
init_logging,
|
||||||
set_verbose_logging,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
from ..utils import get_artifact_folder
|
from ..utils import get_artifact_folder
|
||||||
@@ -123,29 +122,3 @@ class TestInitLogging:
|
|||||||
sum(isinstance(handler, MVTLogHandler) for handler in log.handlers)
|
sum(isinstance(handler, MVTLogHandler) for handler in log.handlers)
|
||||||
== handler_count
|
== handler_count
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_verbose_logging_finds_the_console_handler_among_others(self):
|
|
||||||
# Something else may have attached a handler to the "mvt" logger
|
|
||||||
# before MVT did, so the console handler is not always the first.
|
|
||||||
log = logging.getLogger("mvt")
|
|
||||||
init_logging()
|
|
||||||
foreign_handler = logging.NullHandler()
|
|
||||||
foreign_handler.setLevel(logging.CRITICAL)
|
|
||||||
log.handlers.insert(0, foreign_handler)
|
|
||||||
|
|
||||||
try:
|
|
||||||
set_verbose_logging(True)
|
|
||||||
console_handlers = [
|
|
||||||
handler
|
|
||||||
for handler in log.handlers
|
|
||||||
if isinstance(handler, MVTLogHandler)
|
|
||||||
]
|
|
||||||
assert console_handlers
|
|
||||||
assert all(handler.level == logging.DEBUG for handler in console_handlers)
|
|
||||||
assert foreign_handler.level == logging.CRITICAL
|
|
||||||
|
|
||||||
set_verbose_logging(False)
|
|
||||||
assert all(handler.level == logging.INFO for handler in console_handlers)
|
|
||||||
assert foreign_handler.level == logging.CRITICAL
|
|
||||||
finally:
|
|
||||||
log.handlers.remove(foreign_handler)
|
|
||||||
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user