mirror of
https://github.com/mvt-project/mvt.git
synced 2026-10-31 10:16:21 +01:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
237f77bf7f | ||
|
|
30c11f68c7 | ||
|
|
a92ec7f963 | ||
|
|
10a5bccbe0 | ||
|
|
3b21c9347c | ||
|
|
08e076b71e | ||
|
|
0fdc2c34b0 | ||
|
|
0ee25edf0a | ||
|
|
fa24b5465b | ||
|
|
683b8ba133 | ||
|
|
7b64463727 | ||
|
|
47fd771b2a | ||
|
|
8123db26d9 | ||
|
|
d92a60c9be | ||
|
|
0b48d9fe1d | ||
|
|
067f053627 | ||
|
|
93b7fb5232 | ||
|
|
8617e0bf54 | ||
|
|
53fb12aee8 | ||
|
|
8140e350f7 | ||
|
|
f483223e23 | ||
|
|
2dfe3cbcb1 | ||
|
|
3eff0c550d | ||
|
|
797411e1e5 | ||
|
|
84df51c518 | ||
|
|
a6c3a805d8 |
No files matched your search
@@ -11,7 +11,7 @@ jobs:
|
|||||||
name: Add issue to project
|
name: Add issue to project
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/add-to-project@v0.5.0
|
- uses: actions/add-to-project@v2
|
||||||
with:
|
with:
|
||||||
# You can target a project in a different organization
|
# You can target a project in a different organization
|
||||||
# to the issue
|
# to the issue
|
||||||
|
|||||||
@@ -11,13 +11,13 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: "3.10"
|
python-version: "3.10"
|
||||||
- name: Install uv
|
- name: Install uv
|
||||||
uses: astral-sh/setup-uv@v7
|
uses: astral-sh/setup-uv@v10.0.0
|
||||||
with:
|
with:
|
||||||
enable-cache: true
|
enable-cache: true
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
|
|||||||
@@ -36,10 +36,10 @@ jobs:
|
|||||||
tag-suffix: "-android"
|
tag-suffix: "-android"
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
|
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
|
||||||
- name: Log in to the Container registry
|
- name: Log in to the Container registry
|
||||||
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
@@ -47,7 +47,7 @@ jobs:
|
|||||||
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
|
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
|
||||||
- name: Extract metadata (tags, labels) for Docker
|
- name: Extract metadata (tags, labels) for Docker
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@v5
|
uses: docker/metadata-action@v6
|
||||||
with:
|
with:
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
flavor: |
|
flavor: |
|
||||||
@@ -59,21 +59,24 @@ jobs:
|
|||||||
type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }}
|
type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }}
|
||||||
type=sha,suffix=${{ matrix.platform.tag-suffix }}
|
type=sha,suffix=${{ matrix.platform.tag-suffix }}
|
||||||
type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }}
|
type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }}
|
||||||
|
# Register emulators so Buildx can build the ARM64 images on the AMD64 runner.
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v4
|
||||||
# This step sets up some additional capabilities to generate the provenance and sbom attestations
|
# This step sets up some additional capabilities to generate the provenance and sbom attestations
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v4
|
||||||
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
|
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
|
||||||
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
|
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
|
||||||
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
|
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
|
||||||
- name: Build and push Docker image
|
- name: Build and push Docker image
|
||||||
id: push
|
id: push
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@v7
|
||||||
with:
|
with:
|
||||||
file: ${{ matrix.platform.dockerfile }}
|
file: ${{ matrix.platform.dockerfile }}
|
||||||
context: .
|
context: .
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
push: true
|
push: true
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
provenance: mode=max
|
provenance: mode=max
|
||||||
sbom: true
|
sbom: true
|
||||||
|
|
||||||
@@ -12,13 +12,13 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: "3.10"
|
python-version: "3.10"
|
||||||
- name: Install uv
|
- name: Install uv
|
||||||
uses: astral-sh/setup-uv@v7
|
uses: astral-sh/setup-uv@v10.0.0
|
||||||
with:
|
with:
|
||||||
enable-cache: true
|
enable-cache: true
|
||||||
- name: Install Dependencies
|
- name: Install Dependencies
|
||||||
|
|||||||
@@ -15,13 +15,13 @@ jobs:
|
|||||||
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v7
|
||||||
- name: Install uv
|
- name: Install uv
|
||||||
uses: astral-sh/setup-uv@v7
|
uses: astral-sh/setup-uv@v10.0.0
|
||||||
with:
|
with:
|
||||||
enable-cache: true
|
enable-cache: true
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
- name: Install Python dependencies
|
- name: Install Python dependencies
|
||||||
|
|||||||
@@ -11,10 +11,12 @@ jobs:
|
|||||||
update-ios-version:
|
update-ios-version:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v7
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v7
|
||||||
|
- name: Install script dependencies
|
||||||
|
run: python -m pip install "packaging==26.3"
|
||||||
- name: Run script to fetch latest iOS releases from Apple RSS feed.
|
- name: Run script to fetch latest iOS releases from Apple RSS feed.
|
||||||
run: python3 .github/workflows/scripts/update-ios-releases.py
|
run: python .github/workflows/scripts/update-ios-releases.py
|
||||||
- name: Create Pull Request
|
- name: Create Pull Request
|
||||||
uses: peter-evans/create-pull-request@v8
|
uses: peter-evans/create-pull-request@v8
|
||||||
with:
|
with:
|
||||||
|
|||||||
+13
-2
@@ -119,12 +119,23 @@ ARG DEBIAN_FRONTEND=noninteractive
|
|||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y \
|
&& apt-get install -y \
|
||||||
adb \
|
adb \
|
||||||
|
binutils \
|
||||||
default-jre-headless \
|
default-jre-headless \
|
||||||
|
file \
|
||||||
|
jq \
|
||||||
|
less \
|
||||||
libcurl4 \
|
libcurl4 \
|
||||||
|
libimage-exiftool-perl \
|
||||||
libssl3 \
|
libssl3 \
|
||||||
libusb-1.0-0 \
|
libusb-1.0-0 \
|
||||||
|
moreutils \
|
||||||
|
p7zip-full \
|
||||||
python3 \
|
python3 \
|
||||||
sqlite3
|
ripgrep \
|
||||||
|
sqlite3 \
|
||||||
|
tree \
|
||||||
|
unzip \
|
||||||
|
xxd
|
||||||
COPY --from=build-libplist /build /
|
COPY --from=build-libplist /build /
|
||||||
COPY --from=build-libimobiledevice-glue /build /
|
COPY --from=build-libimobiledevice-glue /build /
|
||||||
COPY --from=build-libtatsu /build /
|
COPY --from=build-libtatsu /build /
|
||||||
@@ -144,7 +155,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2
|
|||||||
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
|
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
|
||||||
|
|
||||||
# Generate adb key folder
|
# Generate adb key folder
|
||||||
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc
|
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc
|
||||||
RUN mkdir /root/.android
|
RUN mkdir /root/.android
|
||||||
|
|
||||||
# Setup investigations environment
|
# Setup investigations environment
|
||||||
|
|||||||
+1
-1
@@ -31,7 +31,7 @@ ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2
|
|||||||
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
|
RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
|
||||||
|
|
||||||
# Generate adb key folder
|
# Generate adb key folder
|
||||||
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey 2&>1 > /dev/null; fi' >> ~/.bashrc
|
RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc
|
||||||
RUN mkdir /root/.android
|
RUN mkdir /root/.android
|
||||||
|
|
||||||
ENTRYPOINT [ "/usr/local/bin/mvt-android" ]
|
ENTRYPOINT [ "/usr/local/bin/mvt-android" ]
|
||||||
@@ -82,6 +82,11 @@ Module-running `check-*` commands can load custom Python modules with
|
|||||||
[development documentation](https://docs.mvt.re/en/latest/development/) for
|
[development documentation](https://docs.mvt.re/en/latest/development/) for
|
||||||
details.
|
details.
|
||||||
|
|
||||||
|
Users can also add top-level commands to `mvt-ios` and `mvt-android` from
|
||||||
|
installed Python packages or local files and folders. See the
|
||||||
|
[custom CLI command documentation](https://docs.mvt.re/en/latest/custom_commands/)
|
||||||
|
for the plugin entry points and `--load-command` interface.
|
||||||
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
@@ -57,3 +57,9 @@ If the backup is encrypted, ABE will prompt you to enter the password.
|
|||||||
Alternatively, [ab-decrypt](https://github.com/joernheissler/ab-decrypt) can be used for that purpose.
|
Alternatively, [ab-decrypt](https://github.com/joernheissler/ab-decrypt) can be used for that purpose.
|
||||||
|
|
||||||
You can then extract SMSs with MVT by passing the folder path as parameter instead of the `.ab` file: `mvt-android check-backup --output /path/to/results/ /path/to/backup/` (the path to backup given should be the folder containing the `apps` folder).
|
You can then extract SMSs with MVT by passing the folder path as parameter instead of the `.ab` file: `mvt-android check-backup --output /path/to/results/ /path/to/backup/` (the path to backup given should be the folder containing the `apps` folder).
|
||||||
|
|
||||||
|
When an output folder is specified, URLs extracted from SMS and MMS messages
|
||||||
|
are also written to `urls.json`. Each entry contains the URL, its expanded
|
||||||
|
destination when MVT resolved a shortened URL during indicator checking, the
|
||||||
|
message timestamp, and the `sms` source. The same file is created by
|
||||||
|
`check-androidqf` when its nested Android backup contains messages with URLs.
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
# Custom CLI Commands
|
||||||
|
|
||||||
|
MVT can load additional top-level commands into `mvt-ios` and `mvt-android`.
|
||||||
|
Custom commands are different from [custom forensic modules](development.md#custom-modules):
|
||||||
|
commands add new CLI operations, while modules add analysis steps to existing
|
||||||
|
`check-*` commands.
|
||||||
|
|
||||||
|
!!! warning
|
||||||
|
|
||||||
|
Custom commands run as trusted Python code inside the MVT process. Install
|
||||||
|
or load commands only from sources you trust. MVT does not sandbox
|
||||||
|
third-party commands, and the MVT maintainers do not maintain them.
|
||||||
|
|
||||||
|
## Install a Command Package
|
||||||
|
|
||||||
|
Python packages can register a Click command or group for either MVT CLI. A
|
||||||
|
minimal package can expose this command from `my_mvt_plugin.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import click
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.argument("path", type=click.Path(exists=True))
|
||||||
|
def summarize(path):
|
||||||
|
"""Summarize an acquisition."""
|
||||||
|
click.echo(f"Summarizing {path}")
|
||||||
|
```
|
||||||
|
|
||||||
|
Register the object in the package's `pyproject.toml`. The entry-point name is
|
||||||
|
the command users invoke:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[project.entry-points."mvt.ios.cli_plugins"]
|
||||||
|
summarize = "my_mvt_plugin:summarize"
|
||||||
|
|
||||||
|
[project.entry-points."mvt.android.cli_plugins"]
|
||||||
|
summarize = "my_mvt_plugin:summarize"
|
||||||
|
```
|
||||||
|
|
||||||
|
Use only the iOS or Android group if the command is platform-specific. After
|
||||||
|
installing the package in the same environment as MVT, it appears directly in
|
||||||
|
the appropriate CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvt-ios summarize ./ios-backup
|
||||||
|
mvt-android summarize ./androidqf-output
|
||||||
|
```
|
||||||
|
|
||||||
|
For a `pipx` installation of MVT, inject the plugin into MVT's environment:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pipx inject mvt my-mvt-plugin
|
||||||
|
```
|
||||||
|
|
||||||
|
When MVT is installed in an active virtual environment, install the plugin with
|
||||||
|
`pip` in that environment.
|
||||||
|
|
||||||
|
## Load a Command File
|
||||||
|
|
||||||
|
For local commands that are not packaged, create a Python file that exports one
|
||||||
|
Click command or group named `cli`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import click
|
||||||
|
|
||||||
|
|
||||||
|
@click.command("case-summary")
|
||||||
|
@click.argument("path", type=click.Path(exists=True))
|
||||||
|
def cli(path):
|
||||||
|
"""Summarize a case directory."""
|
||||||
|
click.echo(f"Summarizing {path}")
|
||||||
|
```
|
||||||
|
|
||||||
|
Pass the file before the custom command name:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvt-ios --load-command ./case_summary.py case-summary ./ios-backup
|
||||||
|
```
|
||||||
|
|
||||||
|
`--load-command` can be repeated and also accepts a folder. MVT loads
|
||||||
|
non-hidden top-level `*.py` files in sorted order and skips `__init__.py`.
|
||||||
|
Every loaded file must export one `cli` object.
|
||||||
|
|
||||||
|
To load a file or folder on every invocation, set the platform-specific
|
||||||
|
environment variable:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export MVT_IOS_CUSTOM_COMMANDS=./ios_commands
|
||||||
|
export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands
|
||||||
|
```
|
||||||
|
|
||||||
|
## Naming and Errors
|
||||||
|
|
||||||
|
Built-in MVT commands cannot be replaced. External command names must also be
|
||||||
|
unique; when installed packages or environment paths collide, MVT keeps the
|
||||||
|
first command and logs a warning. A collision from an explicit
|
||||||
|
`--load-command` is a usage error.
|
||||||
|
|
||||||
|
A package entry point or environment command that cannot be imported appears
|
||||||
|
as a marked broken command without preventing other MVT commands from working.
|
||||||
|
Invoke that command to see its package or file source and the underlying error.
|
||||||
|
An invalid command supplied explicitly with `--load-command` fails immediately
|
||||||
|
with a usage error.
|
||||||
|
|
||||||
|
Installed command packages use the entry-point name as the CLI command name.
|
||||||
|
The entry point must resolve to a `click.Command` or `click.Group`.
|
||||||
+5
-3
@@ -82,14 +82,16 @@ class ExampleCustomModule(MVTModule):
|
|||||||
return None
|
return None
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `supported_commands` to restrict a module to specific platform/command
|
Use `supported_commands` to declare the platform/command pairs a module
|
||||||
pairs. Missing or empty `supported_commands` means the module is available to
|
supports. Empty `supported_commands` means the module will not run and MVT logs
|
||||||
all commands, which keeps older modules compatible. Supported pairs are:
|
a warning. This explicit declaration is required for every command. Supported
|
||||||
|
pairs are:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
("ios", "check-backup")
|
("ios", "check-backup")
|
||||||
("ios", "check-fs")
|
("ios", "check-fs")
|
||||||
("ios", "check-iocs")
|
("ios", "check-iocs")
|
||||||
|
("ios", "check-sysdiagnose")
|
||||||
("android", "check-backup")
|
("android", "check-backup")
|
||||||
("android", "check-bugreport")
|
("android", "check-bugreport")
|
||||||
("android", "check-androidqf")
|
("android", "check-androidqf")
|
||||||
|
|||||||
+15
-1
@@ -31,4 +31,18 @@ Test if the image was created successfully:
|
|||||||
docker run -it mvt
|
docker run -it mvt
|
||||||
```
|
```
|
||||||
|
|
||||||
If a prompt is spawned successfully, you can close it with `exit`.
|
If a prompt is spawned successfully, you can close it with `exit`.
|
||||||
|
|
||||||
|
## Access an iOS device from Docker
|
||||||
|
|
||||||
|
On the Linux host, install and start [usbmuxd](https://github.com/libimobiledevice/usbmuxd), then connect and unlock the iOS device. The daemon exposes the device through the `/var/run/usbmuxd` socket.
|
||||||
|
|
||||||
|
Bind that socket into the container to let MVT communicate with the device:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -it \
|
||||||
|
--mount type=bind,source=/var/run/usbmuxd,target=/var/run/usbmuxd \
|
||||||
|
ghcr.io/mvt-project/mvt
|
||||||
|
```
|
||||||
|
|
||||||
|
If you built the image from source, replace `ghcr.io/mvt-project/mvt` with `mvt`.
|
||||||
+4
-3
@@ -37,8 +37,10 @@ export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
|
|||||||
## Network Access
|
## Network Access
|
||||||
|
|
||||||
When checking URL indicators, MVT follows recognized shortened URLs with an
|
When checking URL indicators, MVT follows recognized shortened URLs with an
|
||||||
HTTP `HEAD` request. The following environment variables control these
|
HTTP `HEAD` request. URL checks are deduplicated and run concurrently, with at
|
||||||
requests:
|
most 20 requests in progress at a time. Redirects within an individual URL
|
||||||
|
chain are still followed sequentially. The following environment variables
|
||||||
|
control these requests:
|
||||||
|
|
||||||
- `MVT_NETWORK_ACCESS_ALLOWED` enables or disables network requests. It defaults
|
- `MVT_NETWORK_ACCESS_ALLOWED` enables or disables network requests. It defaults
|
||||||
to `true`. Set it to `false` to prevent MVT from attempting to resolve
|
to `true`. Set it to `false` to prevent MVT from attempting to resolve
|
||||||
@@ -73,4 +75,3 @@ You can automaticallly download the latest public indicator files with the comma
|
|||||||
|
|
||||||
Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.
|
Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.
|
||||||
|
|
||||||
|
|
||||||
+24
-1
@@ -312,6 +312,15 @@ If indicators are provided through the command-line, they are checked against th
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
### `urls.json`
|
||||||
|
|
||||||
|
This JSON file collects URLs extracted from SMS, iMessage, and WhatsApp
|
||||||
|
messages. Each entry contains the original URL, its expanded destination when
|
||||||
|
MVT resolved a shortened URL during indicator checking, the message timestamp,
|
||||||
|
and its `sms` or `whatsapp` source.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### `sms_attachments.json`
|
### `sms_attachments.json`
|
||||||
|
|
||||||
!!! info "Availability"
|
!!! info "Availability"
|
||||||
@@ -408,7 +417,21 @@ If indicators are provided through the command-line, they are checked against th
|
|||||||
Backup: :material-check:
|
Backup: :material-check:
|
||||||
Full filesystem dump: :material-check:
|
Full filesystem dump: :material-check:
|
||||||
|
|
||||||
This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*.
|
This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*, along with one record per chat session (marked with `"record_type": "chat_session"`) containing the first and last interaction dates of each conversation. Chat sessions produce `chat_first_message` and `chat_last_message` timeline events, and group chats additionally produce a `group_created` event. A chat session's last-message date can postdate its newest stored message when the most recent messages in the chat were deleted.
|
||||||
|
|
||||||
|
Recent WhatsApp versions key 1:1 chat sessions by an opaque LID identifier rather than the contact's phone number. The module resolves these using the `ZWAPHONENUMBERLIDPAIR` table from the *LID.sqlite* database in the same app group (or from *ChatStorage.sqlite* itself in versions that store it there), populating `partner_resolved_phone_number` on chat session records and using the phone number in timeline events. Each LID-phone number pair is also extracted as a record (`"record_type": "lid_phone_number_pair"`) and produces a `lid_pair_recorded` timeline event marking when WhatsApp learned the association.
|
||||||
|
|
||||||
If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*.
|
If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `whatsapp_contacts.json`
|
||||||
|
|
||||||
|
!!! info "Availability"
|
||||||
|
Backup: :material-check:
|
||||||
|
Full filesystem dump: :material-check:
|
||||||
|
|
||||||
|
This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extracts WhatsApp contact records from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ContactsV2.sqlite*, including each contact's phone number, WhatsApp and LID identifiers, and the per-contact disappearing messages timer, which is not recorded in *ChatStorage.sqlite*. Each timestamp stored on a contact record produces a timeline event: `disappearing_mode_set` (when the disappearing messages timer was last changed), `about_changed` (when the contact last changed their "about" text), `about_expiration` (when a timed "about" is scheduled to expire) and `contact_last_updated` (when the contact record was last updated).
|
||||||
|
|
||||||
|
This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup.
|
||||||
|
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# Check an iOS Sysdiagnose
|
||||||
|
|
||||||
|
`mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by
|
||||||
|
custom MVT modules. MVT does not include built-in sysdiagnose modules. You must
|
||||||
|
load at least one custom module that explicitly supports this command.
|
||||||
|
|
||||||
|
The command accepts either an extracted sysdiagnose directory or the original
|
||||||
|
gzip-compressed tar archive.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mvt-ios check-sysdiagnose \
|
||||||
|
--load-module ./sysdiagnose_modules.py \
|
||||||
|
--output ./results \
|
||||||
|
./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--hashes` to include hashes for analyzed files in `info.json`, and
|
||||||
|
`--list-modules` to display the eligible custom modules without running them.
|
||||||
|
|
||||||
|
## Writing a custom module
|
||||||
|
|
||||||
|
Extend `SysdiagnoseExtraction` to access the archive contents consistently for
|
||||||
|
both directory and tar inputs. Each module must declare the command explicitly
|
||||||
|
in `supported_commands`.
|
||||||
|
|
||||||
|
```python
|
||||||
|
from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
|
||||||
|
|
||||||
|
|
||||||
|
class ExampleSysdiagnoseModule(SysdiagnoseExtraction):
|
||||||
|
supported_commands = (("ios", "check-sysdiagnose"),)
|
||||||
|
slug = "example_sysdiagnose"
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
paths = self._get_files_by_pattern("*/example.log")
|
||||||
|
if paths:
|
||||||
|
content = self._get_file_content(paths[0]).decode("utf-8", "replace")
|
||||||
|
self.results = [{"content": content}]
|
||||||
|
|
||||||
|
def check_indicators(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def serialize(self, result):
|
||||||
|
return None
|
||||||
|
```
|
||||||
|
|
||||||
|
The base class provides `from_sysdiagnose_folder()` and
|
||||||
|
`from_sysdiagnose_tar()` setup hooks, as well as protected file lookup, file
|
||||||
|
reading, and timezone extraction helpers. IPS crash-report metadata is exposed
|
||||||
|
on `ips_files`.
|
||||||
+7
-1
@@ -3,6 +3,8 @@ repo_url: https://github.com/mvt-project/mvt
|
|||||||
edit_uri: edit/main/docs/
|
edit_uri: edit/main/docs/
|
||||||
copyright: Copyright © 2021-2023 MVT Project Developers
|
copyright: Copyright © 2021-2023 MVT Project Developers
|
||||||
site_description: Mobile Verification Toolkit Documentation
|
site_description: Mobile Verification Toolkit Documentation
|
||||||
|
not_in_nav: |
|
||||||
|
android/adb.md
|
||||||
markdown_extensions:
|
markdown_extensions:
|
||||||
- attr_list
|
- attr_list
|
||||||
- admonition
|
- admonition
|
||||||
@@ -28,6 +30,7 @@ nav:
|
|||||||
- Welcome: "index.md"
|
- Welcome: "index.md"
|
||||||
- Introduction: "introduction.md"
|
- Introduction: "introduction.md"
|
||||||
- Installation: "install.md"
|
- Installation: "install.md"
|
||||||
|
- Command Completion: "command_completion.md"
|
||||||
- Using Docker: "docker.md"
|
- Using Docker: "docker.md"
|
||||||
- MVT for iOS:
|
- MVT for iOS:
|
||||||
- iOS Forensic Methodology: "ios/methodology.md"
|
- iOS Forensic Methodology: "ios/methodology.md"
|
||||||
@@ -39,11 +42,14 @@ nav:
|
|||||||
- Check a Filesystem Dump:
|
- Check a Filesystem Dump:
|
||||||
- Dumping the filesystem: "ios/filesystem/dump.md"
|
- Dumping the filesystem: "ios/filesystem/dump.md"
|
||||||
- Check a Filesystem Dump with mvt-ios: "ios/filesystem/check.md"
|
- Check a Filesystem Dump with mvt-ios: "ios/filesystem/check.md"
|
||||||
|
- Check a Sysdiagnose: "ios/sysdiagnose.md"
|
||||||
- Records extracted by mvt-ios: "ios/records.md"
|
- Records extracted by mvt-ios: "ios/records.md"
|
||||||
- MVT for Android:
|
- MVT for Android:
|
||||||
- Android Forensic Methodology: "android/methodology.md"
|
- Android Forensic Methodology: "android/methodology.md"
|
||||||
- Check an Android Backup (SMS messages): "android/backup.md"
|
- Check an Android Backup (SMS messages): "android/backup.md"
|
||||||
- Check Android Intrusion Logs: "android/intrusion_logs.md"
|
- Check Android Intrusion Logs: "android/intrusion_logs.md"
|
||||||
- Indicators of Compromise: "iocs.md"
|
- Indicators of Compromise: "iocs.md"
|
||||||
- Development: "development.md"
|
- Development:
|
||||||
|
- Development Instructions: "development.md"
|
||||||
|
- Custom CLI Commands: "custom_commands.md"
|
||||||
- License: "license.md"
|
- License: "license.md"
|
||||||
+13
-13
@@ -17,25 +17,25 @@ classifiers = [
|
|||||||
"Programming Language :: Python",
|
"Programming Language :: Python",
|
||||||
]
|
]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"click==8.3.3",
|
"click==8.4.2",
|
||||||
"rich==14.3.3",
|
"rich==15.0.0",
|
||||||
"tld==0.13.2",
|
"tld==0.13.2",
|
||||||
"requests==2.33.1",
|
"requests==2.34.2",
|
||||||
"simplejson==3.20.2",
|
"simplejson==4.1.1",
|
||||||
"packaging==26.2",
|
"packaging==26.3",
|
||||||
"appdirs==1.4.4",
|
"appdirs==1.4.4",
|
||||||
"iOSbackup==0.9.925",
|
"iOSbackup==0.9.925",
|
||||||
"adb-shell[usb]==0.4.4",
|
"adb-shell[usb]==0.4.4",
|
||||||
"libusb1==3.3.1",
|
"libusb1==3.4.0",
|
||||||
"cryptography==47.0.0",
|
"cryptography==50.0.0",
|
||||||
"PyYAML>=6.0.2",
|
"PyYAML>=6.0.2",
|
||||||
"pyahocorasick==2.3.1",
|
"pyahocorasick==2.3.1",
|
||||||
"betterproto2==0.9.1",
|
"betterproto2==0.10.0",
|
||||||
"pydantic==2.13.3",
|
"pydantic==2.13.4",
|
||||||
"pydantic-settings==2.14.0",
|
"pydantic-settings==2.15.0",
|
||||||
"NSKeyedUnArchiver==1.5.2",
|
"NSKeyedUnArchiver==1.5.2",
|
||||||
"python-dateutil==2.9.0.post0",
|
"python-dateutil==2.9.0.post0",
|
||||||
"tzdata==2026.2",
|
"tzdata==2026.3",
|
||||||
]
|
]
|
||||||
requires-python = ">= 3.10"
|
requires-python = ">= 3.10"
|
||||||
|
|
||||||
@@ -65,9 +65,9 @@ dev = [
|
|||||||
docs = [
|
docs = [
|
||||||
"mkdocs==1.6.1",
|
"mkdocs==1.6.1",
|
||||||
"mkdocs-autorefs==1.4.4",
|
"mkdocs-autorefs==1.4.4",
|
||||||
"mkdocs-material==9.7.6",
|
"mkdocs-material==9.7.7",
|
||||||
"mkdocs-material-extensions==1.3.1",
|
"mkdocs-material-extensions==1.3.1",
|
||||||
"mkdocstrings==1.0.4",
|
"mkdocstrings==1.0.6",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
|
|||||||
@@ -31,21 +31,38 @@ class DumpsysBatteryHistoryArtifact(AndroidArtifact):
|
|||||||
if line.strip() == "":
|
if line.strip() == "":
|
||||||
break
|
break
|
||||||
|
|
||||||
time_elapsed = line.strip().split(" ", 1)[0]
|
time_parts = line.strip().split()
|
||||||
|
time_elapsed = time_parts[0]
|
||||||
|
if (
|
||||||
|
len(time_parts) > 1
|
||||||
|
and len(time_parts[0]) == 5
|
||||||
|
and time_parts[0][2] == "-"
|
||||||
|
and ":" in time_parts[1]
|
||||||
|
):
|
||||||
|
time_elapsed = " ".join(time_parts[:2])
|
||||||
|
|
||||||
event = ""
|
event = ""
|
||||||
if line.find("+job") > 0:
|
if line.find("+job") > 0:
|
||||||
event = "start_job"
|
event = "start_job"
|
||||||
uid = line[line.find("+job") + 5 : line.find(":")]
|
payload = line.split("+job=", 1)[1]
|
||||||
service = line[line.find(":") + 1 :].strip('"')
|
uid, separator, service = payload.partition(":")
|
||||||
|
if not separator:
|
||||||
|
continue
|
||||||
|
service = service.strip().strip('"')
|
||||||
package_name = service.split("/")[0]
|
package_name = service.split("/")[0]
|
||||||
elif line.find("-job") > 0:
|
elif line.find("-job") > 0:
|
||||||
event = "end_job"
|
event = "end_job"
|
||||||
uid = line[line.find("-job") + 5 : line.find(":")]
|
payload = line.split("-job=", 1)[1]
|
||||||
service = line[line.find(":") + 1 :].strip('"')
|
uid, separator, service = payload.partition(":")
|
||||||
|
if not separator:
|
||||||
|
continue
|
||||||
|
service = service.strip().strip('"')
|
||||||
package_name = service.split("/")[0]
|
package_name = service.split("/")[0]
|
||||||
elif line.find("+running +wake_lock=") > 0:
|
elif line.find("+running +wake_lock=") > 0:
|
||||||
uid = line[line.find("+running +wake_lock=") + 21 : line.find(":")]
|
payload = line.split("+running +wake_lock=", 1)[1]
|
||||||
|
uid, separator, _ = payload.partition(":")
|
||||||
|
if not separator:
|
||||||
|
continue
|
||||||
event = "wake"
|
event = "wake"
|
||||||
service = (
|
service = (
|
||||||
line[line.find("*walarm*:") + 9 :].split(" ")[0].strip('"').strip()
|
line[line.find("*walarm*:") + 9 :].split(" ")[0].strip('"').strip()
|
||||||
|
|||||||
@@ -29,10 +29,9 @@ class DumpsysDBInfoArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
def parse(self, output: str) -> None:
|
def parse(self, output: str) -> None:
|
||||||
rxp = re.compile(
|
rxp = re.compile(
|
||||||
r".*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\].*\[Pid:\((\d+)\)\](\w+).*sql\=\"(.+?)\""
|
r".*\[((?:[0-9]{4}-)?[0-9]{2}-[0-9]{2} "
|
||||||
) # pylint: disable=line-too-long
|
r"[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\]\s*"
|
||||||
rxp_no_pid = re.compile(
|
r"(?:\[Pid:\((\d+)\)\])?([\w-]+).*?sql=\"(.+?)\""
|
||||||
r".*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\][ ]{1}(\w+).*sql\=\"(.+?)\""
|
|
||||||
) # pylint: disable=line-too-long
|
) # pylint: disable=line-too-long
|
||||||
|
|
||||||
pool = None
|
pool = None
|
||||||
@@ -56,29 +55,16 @@ class DumpsysDBInfoArtifact(AndroidArtifact):
|
|||||||
pool = None
|
pool = None
|
||||||
continue
|
continue
|
||||||
|
|
||||||
matches = rxp.findall(line)
|
match = rxp.match(line)
|
||||||
if not matches:
|
if not match:
|
||||||
matches = rxp_no_pid.findall(line)
|
continue
|
||||||
if not matches:
|
|
||||||
continue
|
|
||||||
|
|
||||||
match = matches[0]
|
result = {
|
||||||
self.results.append(
|
"isodate": match.group(1),
|
||||||
{
|
"action": match.group(3),
|
||||||
"isodate": match[0],
|
"sql": match.group(4),
|
||||||
"action": match[1],
|
"path": pool,
|
||||||
"sql": match[2],
|
}
|
||||||
"path": pool,
|
if match.group(2):
|
||||||
}
|
result["pid"] = match.group(2)
|
||||||
)
|
self.results.append(result)
|
||||||
else:
|
|
||||||
match = matches[0]
|
|
||||||
self.results.append(
|
|
||||||
{
|
|
||||||
"isodate": match[0],
|
|
||||||
"pid": match[1],
|
|
||||||
"action": match[2],
|
|
||||||
"sql": match[3],
|
|
||||||
"path": pool,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from mvt.android.utils import ROOT_PACKAGES
|
from mvt.android.utils import ROOT_PACKAGES
|
||||||
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
|
||||||
@@ -71,6 +71,8 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
|||||||
"timestamp": "",
|
"timestamp": "",
|
||||||
"first_install_time": "",
|
"first_install_time": "",
|
||||||
"last_update_time": "",
|
"last_update_time": "",
|
||||||
|
"installer": "",
|
||||||
|
"system": False,
|
||||||
"permissions": list(),
|
"permissions": list(),
|
||||||
"requested_permissions": list(),
|
"requested_permissions": list(),
|
||||||
}
|
}
|
||||||
@@ -78,7 +80,14 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
|||||||
in_runtime_permissions = False
|
in_runtime_permissions = False
|
||||||
in_declared_permissions = False
|
in_declared_permissions = False
|
||||||
in_requested_permissions = True
|
in_requested_permissions = True
|
||||||
|
current_user: Optional[int] = None
|
||||||
|
first_install_times: Dict[Optional[int], str] = {}
|
||||||
|
runtime_permissions: Dict[Optional[int], List[Dict[str, Any]]] = {}
|
||||||
for line in output.splitlines():
|
for line in output.splitlines():
|
||||||
|
user_match = re.match(r"User (\d+):", line.strip())
|
||||||
|
if user_match:
|
||||||
|
current_user = int(user_match.group(1))
|
||||||
|
|
||||||
if in_install_permissions:
|
if in_install_permissions:
|
||||||
if line.startswith(" " * 4) and not line.startswith(" " * 6):
|
if line.startswith(" " * 4) and not line.startswith(" " * 6):
|
||||||
in_install_permissions = False
|
in_install_permissions = False
|
||||||
@@ -102,7 +111,7 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
|||||||
if "granted=" in lineinfo[1]:
|
if "granted=" in lineinfo[1]:
|
||||||
granted = "granted=true" in lineinfo[1]
|
granted = "granted=true" in lineinfo[1]
|
||||||
|
|
||||||
details["permissions"].append(
|
runtime_permissions.setdefault(current_user, []).append(
|
||||||
{"name": permission, "granted": granted, "type": "runtime"}
|
{"name": permission, "granted": granted, "type": "runtime"}
|
||||||
)
|
)
|
||||||
if in_declared_permissions:
|
if in_declared_permissions:
|
||||||
@@ -126,8 +135,12 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
|||||||
details["version_code"] = line.split("=", 1)[1].strip()
|
details["version_code"] = line.split("=", 1)[1].strip()
|
||||||
elif line.strip().startswith("timeStamp="):
|
elif line.strip().startswith("timeStamp="):
|
||||||
details["timestamp"] = line.split("=")[1].strip()
|
details["timestamp"] = line.split("=")[1].strip()
|
||||||
|
elif line.strip().startswith("installerPackageName="):
|
||||||
|
details["installer"] = line.split("=", 1)[1].strip()
|
||||||
|
elif line.strip().startswith("pkgFlags="):
|
||||||
|
details["system"] = "SYSTEM" in line.split("=", 1)[1].split()
|
||||||
elif line.strip().startswith("firstInstallTime="):
|
elif line.strip().startswith("firstInstallTime="):
|
||||||
details["first_install_time"] = line.split("=")[1].strip()
|
first_install_times[current_user] = line.split("=", 1)[1].strip()
|
||||||
elif line.strip().startswith("lastUpdateTime="):
|
elif line.strip().startswith("lastUpdateTime="):
|
||||||
details["last_update_time"] = line.split("=")[1].strip()
|
details["last_update_time"] = line.split("=")[1].strip()
|
||||||
elif line.strip() == "install permissions:":
|
elif line.strip() == "install permissions:":
|
||||||
@@ -139,6 +152,19 @@ class DumpsysPackagesArtifact(AndroidArtifact):
|
|||||||
elif line.strip() == "requested permissions:":
|
elif line.strip() == "requested permissions:":
|
||||||
in_requested_permissions = True
|
in_requested_permissions = True
|
||||||
|
|
||||||
|
if 0 in first_install_times:
|
||||||
|
details["first_install_time"] = first_install_times[0]
|
||||||
|
elif None in first_install_times:
|
||||||
|
details["first_install_time"] = first_install_times[None]
|
||||||
|
elif first_install_times:
|
||||||
|
details["first_install_time"] = next(iter(first_install_times.values()))
|
||||||
|
|
||||||
|
if 0 in runtime_permissions:
|
||||||
|
details["permissions"].extend(runtime_permissions[0])
|
||||||
|
elif None in runtime_permissions:
|
||||||
|
details["permissions"].extend(runtime_permissions[None])
|
||||||
|
elif runtime_permissions:
|
||||||
|
details["permissions"].extend(next(iter(runtime_permissions.values())))
|
||||||
return details
|
return details
|
||||||
|
|
||||||
def parse_dumpsys_packages(self, output: str) -> List[Dict[str, Any]]:
|
def parse_dumpsys_packages(self, output: str) -> List[Dict[str, Any]]:
|
||||||
|
|||||||
@@ -96,6 +96,18 @@ class DumpsysReceiversArtifact(AndroidArtifact):
|
|||||||
self.results[intent] = []
|
self.results[intent] = []
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
parts = line.strip().split(" ")
|
||||||
|
if len(parts) < 2:
|
||||||
|
# A single-token line here is not a receiver. Real dumpstate
|
||||||
|
# output can print an action header mis-indented (observed with
|
||||||
|
# 15 leading spaces instead of 6), which used to raise
|
||||||
|
# IndexError and abort the whole module. Treat a trailing-colon
|
||||||
|
# token as the next action, skip anything else.
|
||||||
|
if parts[0].endswith(":"):
|
||||||
|
intent = parts[0][:-1]
|
||||||
|
self.results.setdefault(intent, [])
|
||||||
|
continue
|
||||||
|
|
||||||
# If we are not in an intent block yet, skip.
|
# If we are not in an intent block yet, skip.
|
||||||
if not intent:
|
if not intent:
|
||||||
continue
|
continue
|
||||||
@@ -109,7 +121,7 @@ class DumpsysReceiversArtifact(AndroidArtifact):
|
|||||||
|
|
||||||
# If we got this far, we are processing receivers for the
|
# If we got this far, we are processing receivers for the
|
||||||
# activities we are interested in.
|
# activities we are interested in.
|
||||||
receiver = line.strip().split(" ")[1]
|
receiver = parts[1]
|
||||||
package_name = receiver.split("/")[0]
|
package_name = receiver.split("/")[0]
|
||||||
|
|
||||||
self.results[intent].append(
|
self.results[intent].append(
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ TOMBSTONE_TEXT_KEY_MAPPINGS = {
|
|||||||
"signal": "signal_info",
|
"signal": "signal_info",
|
||||||
"code": "code",
|
"code": "code",
|
||||||
"Cause": "cause",
|
"Cause": "cause",
|
||||||
|
"Abort message": "abort_message",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -67,6 +68,8 @@ class TombstoneCrashResult(pydantic.BaseModel):
|
|||||||
uid: int
|
uid: int
|
||||||
signal_info: SignalInfo
|
signal_info: SignalInfo
|
||||||
cause: Optional[str] = None
|
cause: Optional[str] = None
|
||||||
|
causes: Optional[List[dict]] = None
|
||||||
|
abort_message: Optional[str] = None
|
||||||
extra: Optional[str] = None
|
extra: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
@@ -206,6 +209,11 @@ class TombstoneCrashArtifact(AndroidArtifact):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
def _load_pid_line(self, line: str, tombstone: dict) -> bool:
|
def _load_pid_line(self, line: str, tombstone: dict) -> bool:
|
||||||
|
# The first pid line identifies the crashing thread. Full text tombstones
|
||||||
|
# contain additional pid lines for the other threads in the process.
|
||||||
|
if "pid" in tombstone:
|
||||||
|
return True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
parts = line.split(" >>> ") if " >>> " in line else line.split(">>>")
|
parts = line.split(" >>> ") if " >>> " in line else line.split(">>>")
|
||||||
process_info = parts[0]
|
process_info = parts[0]
|
||||||
|
|||||||
+19
-1
@@ -4,9 +4,16 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from zipfile import BadZipFile
|
||||||
|
|
||||||
import click
|
import click
|
||||||
|
|
||||||
|
from mvt.common.cli_plugins import (
|
||||||
|
ANDROID_CLI_PLUGIN_GROUP,
|
||||||
|
MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||||
|
load_cli_commands_option,
|
||||||
|
register_cli_plugins,
|
||||||
|
)
|
||||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||||
from mvt.common.completion import (
|
from mvt.common.completion import (
|
||||||
SUPPORTED_SHELLS,
|
SUPPORTED_SHELLS,
|
||||||
@@ -81,6 +88,7 @@ def _load_custom_modules(load_module):
|
|||||||
# Main
|
# Main
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@click.group(invoke_without_command=False)
|
@click.group(invoke_without_command=False)
|
||||||
|
@load_cli_commands_option
|
||||||
@click.option(
|
@click.option(
|
||||||
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
||||||
)
|
)
|
||||||
@@ -212,7 +220,10 @@ def check_bugreport(
|
|||||||
|
|
||||||
log.info("Checking Android bug report at path: %s", bugreport_path)
|
log.info("Checking Android bug report at path: %s", bugreport_path)
|
||||||
|
|
||||||
cmd.run()
|
try:
|
||||||
|
cmd.run()
|
||||||
|
except BadZipFile as exc:
|
||||||
|
raise click.ClickException(f"Invalid bugreport archive: {exc}") from exc
|
||||||
cmd.show_alerts_brief()
|
cmd.show_alerts_brief()
|
||||||
cmd.show_support_message()
|
cmd.show_support_message()
|
||||||
|
|
||||||
@@ -501,3 +512,10 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
|||||||
def download_indicators():
|
def download_indicators():
|
||||||
ioc_updates = IndicatorsUpdates()
|
ioc_updates = IndicatorsUpdates()
|
||||||
ioc_updates.update()
|
ioc_updates.update()
|
||||||
|
|
||||||
|
|
||||||
|
register_cli_plugins(
|
||||||
|
cli,
|
||||||
|
entry_point_group=ANDROID_CLI_PLUGIN_GROUP,
|
||||||
|
environment_variable=MVT_ANDROID_CUSTOM_COMMANDS_ENV,
|
||||||
|
)
|
||||||
@@ -3,6 +3,7 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
@@ -94,6 +95,47 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
self.__zip = zipfile.ZipFile(self.target_path)
|
self.__zip = zipfile.ZipFile(self.target_path)
|
||||||
self.__files = self.__zip.namelist()
|
self.__files = self.__zip.namelist()
|
||||||
|
|
||||||
|
self._load_acquisition_context()
|
||||||
|
|
||||||
|
def _load_acquisition_context(self) -> None:
|
||||||
|
"""Pass AndroidQF acquisition metadata to nested commands and modules."""
|
||||||
|
context = {}
|
||||||
|
metadata_files = [
|
||||||
|
file_path
|
||||||
|
for file_path in self.__files
|
||||||
|
if file_path.replace("\\", "/").rsplit("/", 1)[-1] == "acquisition.json"
|
||||||
|
]
|
||||||
|
for file_path in metadata_files:
|
||||||
|
try:
|
||||||
|
metadata = json.loads(self._get_file_content(file_path))
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
context["started"] = metadata.get("started")
|
||||||
|
context["adb_host_public_key"] = metadata.get("adb_host_public_key")
|
||||||
|
break
|
||||||
|
except (json.JSONDecodeError, OSError, TypeError, UnicodeDecodeError):
|
||||||
|
self.log.warning(
|
||||||
|
'Unable to read AndroidQF acquisition metadata "%s"', file_path
|
||||||
|
)
|
||||||
|
|
||||||
|
if not context.get("adb_host_public_key"):
|
||||||
|
key_files = [
|
||||||
|
file_path
|
||||||
|
for file_path in self.__files
|
||||||
|
if file_path.replace("\\", "/").rsplit("/", 1)[-1] == "adb_host_key.pub"
|
||||||
|
]
|
||||||
|
for file_path in key_files:
|
||||||
|
try:
|
||||||
|
context["adb_host_public_key"] = self._get_file_content(
|
||||||
|
file_path
|
||||||
|
).decode("utf-8")
|
||||||
|
break
|
||||||
|
except (OSError, UnicodeDecodeError):
|
||||||
|
self.log.warning(
|
||||||
|
'Unable to read AndroidQF ADB host key "%s"', file_path
|
||||||
|
)
|
||||||
|
|
||||||
|
self.module_options["androidqf_acquisition"] = context
|
||||||
|
|
||||||
def module_init(self, module: AndroidQFModule) -> None: # type: ignore[override]
|
def module_init(self, module: AndroidQFModule) -> None: # type: ignore[override]
|
||||||
if self.__format == "zip" and self.__zip:
|
if self.__format == "zip" and self.__zip:
|
||||||
module.from_zip(self.__zip, self.__files)
|
module.from_zip(self.__zip, self.__files)
|
||||||
@@ -220,6 +262,7 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
cmd.run()
|
cmd.run()
|
||||||
|
|
||||||
self.timeline.extend(cmd.timeline)
|
self.timeline.extend(cmd.timeline)
|
||||||
|
self.url_results.extend(cmd.url_results)
|
||||||
self.alertstore.extend(cmd.alertstore.alerts)
|
self.alertstore.extend(cmd.alertstore.alerts)
|
||||||
finally:
|
finally:
|
||||||
if bugreport:
|
if bugreport:
|
||||||
@@ -257,6 +300,7 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
cmd.run()
|
cmd.run()
|
||||||
|
|
||||||
self.timeline.extend(cmd.timeline)
|
self.timeline.extend(cmd.timeline)
|
||||||
|
self.url_results.extend(cmd.url_results)
|
||||||
self.alertstore.extend(cmd.alertstore.alerts)
|
self.alertstore.extend(cmd.alertstore.alerts)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -296,6 +340,7 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
|
|
||||||
elif self.__format == "zip" and self.__zip:
|
elif self.__format == "zip" and self.__zip:
|
||||||
temp_dir = tempfile.mkdtemp(prefix="mvt_intrusion_logs_")
|
temp_dir = tempfile.mkdtemp(prefix="mvt_intrusion_logs_")
|
||||||
|
temp_root = Path(temp_dir).resolve()
|
||||||
for entry in intrusion_log_files:
|
for entry in intrusion_log_files:
|
||||||
normalized = entry.replace("\\", "/")
|
normalized = entry.replace("\\", "/")
|
||||||
idx = normalized.find("intrusion_logs/")
|
idx = normalized.find("intrusion_logs/")
|
||||||
@@ -303,9 +348,15 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
if not relative or relative.endswith("/"):
|
if not relative or relative.endswith("/"):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
target = os.path.join(temp_dir, relative)
|
target = (temp_root / relative).resolve()
|
||||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
if not target.is_relative_to(temp_root):
|
||||||
with self.__zip.open(entry) as src, open(target, "wb") as dst:
|
self.log.warning(
|
||||||
|
"Skipping unsafe intrusion log archive entry: %s", entry
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with self.__zip.open(entry) as src, target.open("wb") as dst:
|
||||||
dst.write(src.read())
|
dst.write(src.read())
|
||||||
|
|
||||||
intrusion_logs_path = temp_dir
|
intrusion_logs_path = temp_dir
|
||||||
@@ -329,6 +380,7 @@ class CmdAndroidCheckAndroidQF(Command):
|
|||||||
cmd.run()
|
cmd.run()
|
||||||
|
|
||||||
self.timeline.extend(cmd.timeline)
|
self.timeline.extend(cmd.timeline)
|
||||||
|
self.url_results.extend(cmd.url_results)
|
||||||
self.alertstore.extend(cmd.alertstore.alerts)
|
self.alertstore.extend(cmd.alertstore.alerts)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
|
|
||||||
from .aqf_files import AQFFiles
|
from .aqf_files import AQFFiles
|
||||||
from .aqf_getprop import AQFGetProp
|
from .aqf_getprop import AQFGetProp
|
||||||
|
from .aqf_log_timestamps import AQFLogTimestamps
|
||||||
from .aqf_packages import AQFPackages
|
from .aqf_packages import AQFPackages
|
||||||
from .aqf_processes import AQFProcesses
|
from .aqf_processes import AQFProcesses
|
||||||
from .aqf_settings import AQFSettings
|
from .aqf_settings import AQFSettings
|
||||||
@@ -17,6 +18,7 @@ ANDROIDQF_MODULES = [
|
|||||||
AQFGetProp,
|
AQFGetProp,
|
||||||
AQFSettings,
|
AQFSettings,
|
||||||
AQFFiles,
|
AQFFiles,
|
||||||
|
AQFLogTimestamps,
|
||||||
RootBinaries,
|
RootBinaries,
|
||||||
Mounts,
|
Mounts,
|
||||||
]
|
]
|
||||||
@@ -36,6 +36,8 @@ class SMS(BackupModule):
|
|||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
messages = []
|
||||||
|
url_batches = []
|
||||||
for message in self.results:
|
for message in self.results:
|
||||||
if "body" not in message:
|
if "body" not in message:
|
||||||
continue
|
continue
|
||||||
@@ -44,12 +46,21 @@ class SMS(BackupModule):
|
|||||||
if message_links == []:
|
if message_links == []:
|
||||||
message_links = check_for_links(message.get("text", ""))
|
message_links = check_for_links(message.get("text", ""))
|
||||||
|
|
||||||
ioc_match = self.indicators.check_urls(message_links)
|
messages.append(message)
|
||||||
|
url_batches.append(message_links)
|
||||||
|
|
||||||
|
for message, ioc_match in zip(
|
||||||
|
messages, self.indicators.check_url_batches(url_batches)
|
||||||
|
):
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message, "", message, matched_indicator=ioc_match.ioc
|
ioc_match.message, "", message, matched_indicator=ioc_match.ioc
|
||||||
)
|
)
|
||||||
continue
|
|
||||||
|
def collect_url_results(self) -> None:
|
||||||
|
for message in self.results:
|
||||||
|
for url in message.get("links", []):
|
||||||
|
self.add_url_result(url, message.get("isodate"), "sms")
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
sms_path = "apps/com.android.providers.telephony/d_f/*_sms_backup"
|
sms_path = "apps/com.android.providers.telephony/d_f/*_sms_backup"
|
||||||
|
|||||||
@@ -3,11 +3,15 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact
|
from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact
|
||||||
from mvt.common.module_types import ModuleResults
|
from mvt.common.module_types import ModuleResults
|
||||||
|
from mvt.common.utils import convert_datetime_to_iso
|
||||||
|
|
||||||
from .base import BugReportModule
|
from .base import BugReportModule
|
||||||
|
|
||||||
@@ -53,3 +57,122 @@ class DumpsysADBState(DumpsysADBArtifact, BugReportModule):
|
|||||||
"Identified a total of %d trusted ADB keys",
|
"Identified a total of %d trusted ADB keys",
|
||||||
len(self.results[0].get("user_keys", [])),
|
len(self.results[0].get("user_keys", [])),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _key_material(public_key: object) -> str:
|
||||||
|
if isinstance(public_key, bytes):
|
||||||
|
public_key = public_key.decode("utf-8", errors="replace")
|
||||||
|
if not isinstance(public_key, str):
|
||||||
|
return ""
|
||||||
|
return public_key.strip().split(" ", 1)[0]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_valid_key(public_key: str) -> bool:
|
||||||
|
if not public_key:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return bool(base64.b64decode(public_key, validate=True))
|
||||||
|
except (binascii.Error, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_acquisition_time(value: object) -> Optional[datetime.datetime]:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
timestamp = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if not timestamp.tzinfo:
|
||||||
|
timestamp = timestamp.replace(tzinfo=datetime.timezone.utc)
|
||||||
|
return timestamp.astimezone(datetime.timezone.utc)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_last_connected(value: object) -> Optional[datetime.datetime]:
|
||||||
|
try:
|
||||||
|
return datetime.datetime.fromtimestamp(
|
||||||
|
int(str(value)) / 1000,
|
||||||
|
tz=datetime.timezone.utc,
|
||||||
|
)
|
||||||
|
except (OSError, OverflowError, TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _trusted_keys(self) -> list[dict]:
|
||||||
|
"""Return unique trusted keys, preferring keystore connection metadata."""
|
||||||
|
trusted_keys = []
|
||||||
|
seen = set()
|
||||||
|
for result in self.results:
|
||||||
|
keystore = result.get("keystore", [])
|
||||||
|
candidates = keystore if isinstance(keystore, list) else []
|
||||||
|
candidates = [*candidates, *result.get("user_keys", [])]
|
||||||
|
for candidate in candidates:
|
||||||
|
if not isinstance(candidate, dict):
|
||||||
|
continue
|
||||||
|
key = self._key_material(candidate.get("key"))
|
||||||
|
identity = key or repr(candidate)
|
||||||
|
if identity in seen:
|
||||||
|
continue
|
||||||
|
seen.add(identity)
|
||||||
|
trusted_keys.append(candidate)
|
||||||
|
return trusted_keys
|
||||||
|
|
||||||
|
def check_indicators(self) -> None:
|
||||||
|
if "androidqf_acquisition" not in self.module_options:
|
||||||
|
return super().check_indicators()
|
||||||
|
|
||||||
|
context = self.module_options.get("androidqf_acquisition")
|
||||||
|
if not isinstance(context, dict):
|
||||||
|
context = {}
|
||||||
|
acquisition_key = self._key_material(context.get("adb_host_public_key"))
|
||||||
|
if acquisition_key and not self._is_valid_key(acquisition_key):
|
||||||
|
acquisition_key = ""
|
||||||
|
acquisition_time = self._parse_acquisition_time(context.get("started"))
|
||||||
|
cutoff = (
|
||||||
|
acquisition_time - datetime.timedelta(days=1) if acquisition_time else None
|
||||||
|
)
|
||||||
|
|
||||||
|
for trusted_key in self._trusted_keys():
|
||||||
|
key = self._key_material(trusted_key.get("key"))
|
||||||
|
fingerprint = trusted_key.get("fingerprint") or "<unknown key>"
|
||||||
|
user = trusted_key.get("user") or "unknown user"
|
||||||
|
description = f"{fingerprint} ({user})"
|
||||||
|
last_connected = self._parse_last_connected(
|
||||||
|
trusted_key.get("last_connected")
|
||||||
|
)
|
||||||
|
event_time = (
|
||||||
|
convert_datetime_to_iso(last_connected) if last_connected else ""
|
||||||
|
)
|
||||||
|
|
||||||
|
if not self._is_valid_key(key):
|
||||||
|
self.alertstore.low(
|
||||||
|
f"Found an invalid trusted ADB host key: {description}",
|
||||||
|
event_time,
|
||||||
|
trusted_key,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not acquisition_key:
|
||||||
|
self.alertstore.low(
|
||||||
|
"Found a trusted ADB host key, but the AndroidQF acquisition "
|
||||||
|
f"does not include its host key: {description}",
|
||||||
|
event_time,
|
||||||
|
trusted_key,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if key != acquisition_key:
|
||||||
|
self.alertstore.low(
|
||||||
|
"Found a trusted ADB host key different from the AndroidQF "
|
||||||
|
f"acquisition host: {description}",
|
||||||
|
event_time,
|
||||||
|
trusted_key,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if cutoff and last_connected and last_connected <= cutoff:
|
||||||
|
self.alertstore.info(
|
||||||
|
"Found a trusted ADB host key last connected at least one day "
|
||||||
|
f"before the AndroidQF acquisition: {description}",
|
||||||
|
event_time,
|
||||||
|
trusted_key,
|
||||||
|
)
|
||||||
@@ -131,20 +131,23 @@ def decrypt_backup_data(encrypted_backup, password, encryption_algo, format_vers
|
|||||||
if password is None:
|
if password is None:
|
||||||
raise InvalidBackupPassword()
|
raise InvalidBackupPassword()
|
||||||
|
|
||||||
[
|
try:
|
||||||
user_salt,
|
[
|
||||||
checksum_salt,
|
user_salt,
|
||||||
pbkdf2_rounds,
|
checksum_salt,
|
||||||
user_iv,
|
pbkdf2_rounds,
|
||||||
master_key_blob,
|
user_iv,
|
||||||
encrypted_data,
|
master_key_blob,
|
||||||
] = encrypted_backup.split(b"\n", 5)
|
encrypted_data,
|
||||||
|
] = encrypted_backup.split(b"\n", 5)
|
||||||
|
|
||||||
user_salt = bytes.fromhex(user_salt.decode("utf-8"))
|
user_salt = bytes.fromhex(user_salt.decode("utf-8"))
|
||||||
checksum_salt = bytes.fromhex(checksum_salt.decode("utf-8"))
|
checksum_salt = bytes.fromhex(checksum_salt.decode("utf-8"))
|
||||||
pbkdf2_rounds = int(pbkdf2_rounds)
|
pbkdf2_rounds = int(pbkdf2_rounds)
|
||||||
user_iv = bytes.fromhex(user_iv.decode("utf-8"))
|
user_iv = bytes.fromhex(user_iv.decode("utf-8"))
|
||||||
master_key_blob = bytes.fromhex(master_key_blob.decode("utf-8"))
|
master_key_blob = bytes.fromhex(master_key_blob.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, ValueError) as exc:
|
||||||
|
raise AndroidBackupParsingError("Invalid encrypted backup header") from exc
|
||||||
|
|
||||||
# Derive decryption master key from password.
|
# Derive decryption master key from password.
|
||||||
master_key, master_iv = decrypt_master_key(
|
master_key, master_iv = decrypt_master_key(
|
||||||
@@ -174,10 +177,12 @@ def parse_backup_file(data, password=None):
|
|||||||
if not data.startswith(b"ANDROID BACKUP"):
|
if not data.startswith(b"ANDROID BACKUP"):
|
||||||
raise AndroidBackupParsingError("Invalid file header")
|
raise AndroidBackupParsingError("Invalid file header")
|
||||||
|
|
||||||
[_, version, is_compressed, encryption_algo, tar_data] = data.split(b"\n", 4)
|
try:
|
||||||
|
[_, version, is_compressed, encryption_algo, tar_data] = data.split(b"\n", 4)
|
||||||
version = int(version)
|
version = int(version)
|
||||||
is_compressed = int(is_compressed)
|
is_compressed = int(is_compressed)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise AndroidBackupParsingError("Invalid file header") from exc
|
||||||
|
|
||||||
if encryption_algo != b"none":
|
if encryption_algo != b"none":
|
||||||
tar_data = decrypt_backup_data(
|
tar_data = decrypt_backup_data(
|
||||||
|
|||||||
@@ -0,0 +1,297 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2026 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.metadata
|
||||||
|
import importlib.util
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from types import ModuleType
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
IOS_CLI_PLUGIN_GROUP = "mvt.ios.cli_plugins"
|
||||||
|
ANDROID_CLI_PLUGIN_GROUP = "mvt.android.cli_plugins"
|
||||||
|
MVT_IOS_CUSTOM_COMMANDS_ENV = "MVT_IOS_CUSTOM_COMMANDS"
|
||||||
|
MVT_ANDROID_CUSTOM_COMMANDS_ENV = "MVT_ANDROID_CUSTOM_COMMANDS"
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class CustomCommandLoadError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class BrokenPluginCommand(click.Command):
|
||||||
|
"""A placeholder for an installed or configured command that failed to load."""
|
||||||
|
|
||||||
|
def __init__(self, name: str, source: str, exception: BaseException):
|
||||||
|
super().__init__(
|
||||||
|
name,
|
||||||
|
help=(
|
||||||
|
f"Unable to load external command from {source}.\n\n"
|
||||||
|
f"{type(exception).__name__}: {exception}"
|
||||||
|
),
|
||||||
|
short_help="Warning: external command could not be loaded.",
|
||||||
|
)
|
||||||
|
self.source = source
|
||||||
|
self.exception = exception
|
||||||
|
|
||||||
|
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
|
||||||
|
return args
|
||||||
|
|
||||||
|
def invoke(self, ctx: click.Context) -> None:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"Unable to load external command '{self.name}' from {self.source}: "
|
||||||
|
f"{type(self.exception).__name__}: {self.exception}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _module_name_for_path(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:16]
|
||||||
|
return f"_mvt_custom_command_{path.stem}_{digest}"
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_command_files(path: Path) -> Iterable[Path]:
|
||||||
|
if path.is_file():
|
||||||
|
if path.suffix != ".py":
|
||||||
|
raise CustomCommandLoadError(
|
||||||
|
f"Custom command file is not a Python file: {path}"
|
||||||
|
)
|
||||||
|
yield path
|
||||||
|
return
|
||||||
|
|
||||||
|
if path.is_dir():
|
||||||
|
for child in sorted(path.iterdir()):
|
||||||
|
if child.name.startswith(".") or child.name == "__init__.py":
|
||||||
|
continue
|
||||||
|
if child.is_file() and child.suffix == ".py":
|
||||||
|
yield child
|
||||||
|
return
|
||||||
|
|
||||||
|
raise CustomCommandLoadError(f"Custom command path does not exist: {path}")
|
||||||
|
|
||||||
|
|
||||||
|
def _load_python_file(path: Path) -> ModuleType:
|
||||||
|
module_name = _module_name_for_path(path)
|
||||||
|
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||||
|
if spec is None or spec.loader is None:
|
||||||
|
raise CustomCommandLoadError(f"Unable to load custom command file: {path}")
|
||||||
|
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[module_name] = module
|
||||||
|
try:
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
except (Exception, SystemExit) as exc:
|
||||||
|
raise CustomCommandLoadError(
|
||||||
|
f"Unable to import custom command {path}: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def load_cli_command_file(path: Path) -> click.Command:
|
||||||
|
module = _load_python_file(path)
|
||||||
|
command = getattr(module, "cli", None)
|
||||||
|
if not isinstance(command, click.Command):
|
||||||
|
raise CustomCommandLoadError(
|
||||||
|
f"Custom command {path} must export a Click command or group named 'cli'"
|
||||||
|
)
|
||||||
|
if command.name is None:
|
||||||
|
raise CustomCommandLoadError(f"Custom command {path} has no command name")
|
||||||
|
return command
|
||||||
|
|
||||||
|
|
||||||
|
def _register_command(
|
||||||
|
group: click.Group,
|
||||||
|
command: click.Command,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
source: str,
|
||||||
|
collision_is_error: bool,
|
||||||
|
) -> bool:
|
||||||
|
if name in group.commands:
|
||||||
|
registered_sources = getattr(group, "_mvt_external_command_sources", {})
|
||||||
|
if registered_sources.get(name) == source:
|
||||||
|
return False
|
||||||
|
message = (
|
||||||
|
f"Unable to register external command '{name}' from {source}: "
|
||||||
|
"the command name is already registered"
|
||||||
|
)
|
||||||
|
if collision_is_error:
|
||||||
|
raise CustomCommandLoadError(message)
|
||||||
|
log.warning(message)
|
||||||
|
return False
|
||||||
|
|
||||||
|
group.add_command(command, name=name)
|
||||||
|
registered_sources = getattr(group, "_mvt_external_command_sources", {})
|
||||||
|
registered_sources[name] = source
|
||||||
|
setattr(group, "_mvt_external_command_sources", registered_sources)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def register_cli_commands_from_path(
|
||||||
|
group: click.Group,
|
||||||
|
path: str | Path,
|
||||||
|
*,
|
||||||
|
collision_is_error: bool = False,
|
||||||
|
failures_are_errors: bool = False,
|
||||||
|
) -> list[str]:
|
||||||
|
resolved_path = Path(path).expanduser().resolve()
|
||||||
|
registered: list[str] = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
command_files = list(_iter_command_files(resolved_path))
|
||||||
|
except CustomCommandLoadError as exc:
|
||||||
|
if failures_are_errors:
|
||||||
|
raise
|
||||||
|
name = resolved_path.stem.replace("_", "-") or "custom-command"
|
||||||
|
if _register_command(
|
||||||
|
group,
|
||||||
|
BrokenPluginCommand(name, str(resolved_path), exc),
|
||||||
|
name=name,
|
||||||
|
source=str(resolved_path),
|
||||||
|
collision_is_error=collision_is_error,
|
||||||
|
):
|
||||||
|
registered.append(name)
|
||||||
|
return registered
|
||||||
|
|
||||||
|
for command_file in command_files:
|
||||||
|
try:
|
||||||
|
command = load_cli_command_file(command_file)
|
||||||
|
except CustomCommandLoadError as exc:
|
||||||
|
if failures_are_errors:
|
||||||
|
raise
|
||||||
|
name = command_file.stem.replace("_", "-")
|
||||||
|
command = BrokenPluginCommand(name, str(command_file), exc)
|
||||||
|
|
||||||
|
command_name = command.name
|
||||||
|
if command_name is None:
|
||||||
|
raise CustomCommandLoadError(
|
||||||
|
f"Custom command {command_file} has no command name"
|
||||||
|
)
|
||||||
|
if _register_command(
|
||||||
|
group,
|
||||||
|
command,
|
||||||
|
name=command_name,
|
||||||
|
source=str(command_file),
|
||||||
|
collision_is_error=collision_is_error,
|
||||||
|
):
|
||||||
|
registered.append(command_name)
|
||||||
|
|
||||||
|
return registered
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_point_source(entry_point: importlib.metadata.EntryPoint) -> str:
|
||||||
|
try:
|
||||||
|
distribution = getattr(entry_point, "dist", None)
|
||||||
|
if distribution is None:
|
||||||
|
return f"entry point {entry_point.value}"
|
||||||
|
|
||||||
|
name = distribution.metadata.get("Name", "unknown distribution")
|
||||||
|
version = getattr(distribution, "version", None)
|
||||||
|
if version:
|
||||||
|
return f"{name} {version} ({entry_point.value})"
|
||||||
|
return f"{name} ({entry_point.value})"
|
||||||
|
except Exception:
|
||||||
|
return f"entry point {entry_point.value}"
|
||||||
|
|
||||||
|
|
||||||
|
def register_installed_cli_commands(
|
||||||
|
group: click.Group,
|
||||||
|
entry_point_group: str,
|
||||||
|
) -> list[str]:
|
||||||
|
try:
|
||||||
|
entry_points = importlib.metadata.entry_points(group=entry_point_group)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning(
|
||||||
|
"Unable to discover external commands in entry-point group %s: %s",
|
||||||
|
entry_point_group,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
ordered_entry_points = sorted(
|
||||||
|
entry_points,
|
||||||
|
key=lambda entry_point: (
|
||||||
|
entry_point.name,
|
||||||
|
_entry_point_source(entry_point),
|
||||||
|
entry_point.value,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
registered: list[str] = []
|
||||||
|
|
||||||
|
for entry_point in ordered_entry_points:
|
||||||
|
source = _entry_point_source(entry_point)
|
||||||
|
try:
|
||||||
|
command = entry_point.load()
|
||||||
|
if not isinstance(command, click.Command):
|
||||||
|
raise TypeError(
|
||||||
|
f"entry point must resolve to a Click command or group, "
|
||||||
|
f"not {type(command).__name__}"
|
||||||
|
)
|
||||||
|
except (Exception, SystemExit) as exc:
|
||||||
|
command = BrokenPluginCommand(entry_point.name, source, exc)
|
||||||
|
|
||||||
|
if _register_command(
|
||||||
|
group,
|
||||||
|
command,
|
||||||
|
name=entry_point.name,
|
||||||
|
source=source,
|
||||||
|
collision_is_error=False,
|
||||||
|
):
|
||||||
|
registered.append(entry_point.name)
|
||||||
|
|
||||||
|
return registered
|
||||||
|
|
||||||
|
|
||||||
|
def register_cli_plugins(
|
||||||
|
group: click.Group,
|
||||||
|
*,
|
||||||
|
entry_point_group: str,
|
||||||
|
environment_variable: str,
|
||||||
|
) -> None:
|
||||||
|
environment_path = os.environ.get(environment_variable)
|
||||||
|
if environment_path:
|
||||||
|
register_cli_commands_from_path(group, environment_path)
|
||||||
|
register_installed_cli_commands(group, entry_point_group)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_command_option_callback(
|
||||||
|
ctx: click.Context,
|
||||||
|
param: click.Parameter,
|
||||||
|
paths: tuple[str, ...],
|
||||||
|
) -> tuple[str, ...]:
|
||||||
|
if not isinstance(ctx.command, click.Group):
|
||||||
|
raise click.ClickException("--load-command requires a Click command group")
|
||||||
|
|
||||||
|
for path in paths:
|
||||||
|
try:
|
||||||
|
register_cli_commands_from_path(
|
||||||
|
ctx.command,
|
||||||
|
path,
|
||||||
|
collision_is_error=True,
|
||||||
|
failures_are_errors=True,
|
||||||
|
)
|
||||||
|
except CustomCommandLoadError as exc:
|
||||||
|
raise click.BadParameter(str(exc), ctx=ctx, param=param) from exc
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
load_cli_commands_option = click.option(
|
||||||
|
"--load-command",
|
||||||
|
"load_commands",
|
||||||
|
type=click.Path(exists=True),
|
||||||
|
multiple=True,
|
||||||
|
expose_value=False,
|
||||||
|
is_eager=True,
|
||||||
|
callback=_load_command_option_callback,
|
||||||
|
help=(
|
||||||
|
"Load a custom CLI command from a Python file or folder "
|
||||||
|
"(can be invoked multiple times)"
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -20,7 +20,7 @@ from .config import settings
|
|||||||
from .indicators import Indicators
|
from .indicators import Indicators
|
||||||
from .module import EncryptedBackupError, MVTModule, run_module, save_timeline
|
from .module import EncryptedBackupError, MVTModule, run_module, save_timeline
|
||||||
from .module_loader import module_supports_command
|
from .module_loader import module_supports_command
|
||||||
from .module_types import ModuleTimeline
|
from .module_types import ModuleTimeline, URLResult
|
||||||
from .utils import (
|
from .utils import (
|
||||||
CustomJSONEncoder,
|
CustomJSONEncoder,
|
||||||
convert_datetime_to_iso,
|
convert_datetime_to_iso,
|
||||||
@@ -73,6 +73,7 @@ class Command:
|
|||||||
self.hashes = hashes
|
self.hashes = hashes
|
||||||
self.hash_values: list[dict[str, Any]] = []
|
self.hash_values: list[dict[str, Any]] = []
|
||||||
self.timeline: ModuleTimeline = []
|
self.timeline: ModuleTimeline = []
|
||||||
|
self.url_results: list[URLResult] = []
|
||||||
|
|
||||||
# Load IOCs
|
# Load IOCs
|
||||||
self._create_storage()
|
self._create_storage()
|
||||||
@@ -150,6 +151,14 @@ class Command:
|
|||||||
with open(alerts_path, "w+", encoding="utf-8") as handle:
|
with open(alerts_path, "w+", encoding="utf-8") as handle:
|
||||||
json.dump(alerts, handle, indent=4, cls=CustomJSONEncoder)
|
json.dump(alerts, handle, indent=4, cls=CustomJSONEncoder)
|
||||||
|
|
||||||
|
def _store_urls(self) -> None:
|
||||||
|
if not self.results_path or not self.url_results:
|
||||||
|
return
|
||||||
|
|
||||||
|
urls_path = os.path.join(self.results_path, "urls.json")
|
||||||
|
with open(urls_path, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(self.url_results, handle, indent=4, cls=CustomJSONEncoder)
|
||||||
|
|
||||||
def _store_alerts_timeline(self) -> None:
|
def _store_alerts_timeline(self) -> None:
|
||||||
if not self.results_path:
|
if not self.results_path:
|
||||||
return
|
return
|
||||||
@@ -396,6 +405,7 @@ class Command:
|
|||||||
self.executed.append(m)
|
self.executed.append(m)
|
||||||
executed_by_type[module] = m
|
executed_by_type[module] = m
|
||||||
self.timeline.extend(m.timeline)
|
self.timeline.extend(m.timeline)
|
||||||
|
self.url_results.extend(m.url_results)
|
||||||
self.alertstore.extend(m.alertstore.alerts)
|
self.alertstore.extend(m.alertstore.alerts)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -410,4 +420,5 @@ class Command:
|
|||||||
self._store_timeline()
|
self._store_timeline()
|
||||||
self._store_alerts_timeline()
|
self._store_alerts_timeline()
|
||||||
self._store_alerts()
|
self._store_alerts()
|
||||||
|
self._store_urls()
|
||||||
self._store_info()
|
self._store_info()
|
||||||
@@ -38,6 +38,7 @@ HELP_MSG_BACKUP_KEYFILE = (
|
|||||||
HELP_MSG_EXTRACT_KEY = "Extract decryption key from an iTunes backup"
|
HELP_MSG_EXTRACT_KEY = "Extract decryption key from an iTunes backup"
|
||||||
HELP_MSG_CHECK_IOS_BACKUP = "Extract artifacts from an iTunes backup"
|
HELP_MSG_CHECK_IOS_BACKUP = "Extract artifacts from an iTunes backup"
|
||||||
HELP_MSG_CHECK_FS = "Extract artifacts from a full filesystem dump"
|
HELP_MSG_CHECK_FS = "Extract artifacts from a full filesystem dump"
|
||||||
|
HELP_MSG_CHECK_SYSDIAGNOSE = "Extract artifacts from an iOS sysdiagnose archive"
|
||||||
|
|
||||||
# Android Specific
|
# Android Specific
|
||||||
HELP_MSG_ANDROID_BACKUP_PASSWORD = "The backup password to use for an Android backup"
|
HELP_MSG_ANDROID_BACKUP_PASSWORD = "The backup password to use for an Android backup"
|
||||||
|
|||||||
@@ -7,9 +7,10 @@ import glob
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from typing import Any, Dict, Iterator, List, Optional
|
from typing import Any, Dict, Iterator, List, Optional, Sequence
|
||||||
|
|
||||||
import ahocorasick
|
import ahocorasick
|
||||||
from appdirs import user_data_dir
|
from appdirs import user_data_dir
|
||||||
@@ -22,6 +23,8 @@ MVT_INDICATORS_FOLDER = os.path.join(MVT_DATA_FOLDER, "indicators")
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
URL_CHECK_MAX_WORKERS = 20
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Indicator:
|
class Indicator:
|
||||||
@@ -46,6 +49,7 @@ class Indicators:
|
|||||||
self.log = log
|
self.log = log
|
||||||
self.ioc_collections: List[Dict[str, Any]] = []
|
self.ioc_collections: List[Dict[str, Any]] = []
|
||||||
self.total_ioc_count = 0
|
self.total_ioc_count = 0
|
||||||
|
self.resolved_urls: Dict[str, str] = {}
|
||||||
|
|
||||||
def _load_downloaded_indicators(self) -> None:
|
def _load_downloaded_indicators(self) -> None:
|
||||||
if not os.path.isdir(MVT_INDICATORS_FOLDER):
|
if not os.path.isdir(MVT_INDICATORS_FOLDER):
|
||||||
@@ -436,9 +440,14 @@ class Indicators:
|
|||||||
orig_url.url,
|
orig_url.url,
|
||||||
dest_url.url,
|
dest_url.url,
|
||||||
)
|
)
|
||||||
return self.check_url(dest_url.url)
|
match = self.check_url(dest_url.url)
|
||||||
|
self.resolved_urls[url] = self.resolved_urls.get(
|
||||||
|
dest_url.url, dest_url.url
|
||||||
|
)
|
||||||
|
return match
|
||||||
|
|
||||||
final_url = dest_url
|
final_url = dest_url
|
||||||
|
self.resolved_urls[url] = final_url.url
|
||||||
else:
|
else:
|
||||||
# If it's not shortened, we just use the original URL object.
|
# If it's not shortened, we just use the original URL object.
|
||||||
final_url = orig_url
|
final_url = orig_url
|
||||||
@@ -479,6 +488,13 @@ class Indicators:
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_expanded_url(self, url: str) -> Optional[str]:
|
||||||
|
"""Return the final URL recorded while checking a shortened URL."""
|
||||||
|
expanded_url = self.resolved_urls.get(url)
|
||||||
|
if expanded_url and expanded_url != url:
|
||||||
|
return expanded_url
|
||||||
|
return None
|
||||||
|
|
||||||
def check_urls(self, urls: list) -> Optional[IndicatorMatch]:
|
def check_urls(self, urls: list) -> Optional[IndicatorMatch]:
|
||||||
"""Check a list of URLs against the provided list of domain indicators.
|
"""Check a list of URLs against the provided list of domain indicators.
|
||||||
|
|
||||||
@@ -490,12 +506,41 @@ class Indicators:
|
|||||||
if not urls:
|
if not urls:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
for url in urls:
|
return self.check_url_batches([urls])[0]
|
||||||
check = self.check_url(url)
|
|
||||||
if check:
|
|
||||||
return check
|
|
||||||
|
|
||||||
return None
|
def check_url_batches(
|
||||||
|
self, url_batches: Sequence[Optional[Sequence[str]]]
|
||||||
|
) -> List[Optional[IndicatorMatch]]:
|
||||||
|
"""Check batches of URLs concurrently while preserving batch order.
|
||||||
|
|
||||||
|
URLs are deduplicated across batches before checking. Each returned item
|
||||||
|
is the first indicator match from the corresponding input batch, using
|
||||||
|
the original URL order.
|
||||||
|
"""
|
||||||
|
batches = [list(urls) if urls else [] for urls in url_batches]
|
||||||
|
unique_urls = list(
|
||||||
|
dict.fromkeys(url for urls in batches for url in urls)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not unique_urls:
|
||||||
|
return [None] * len(batches)
|
||||||
|
|
||||||
|
if settings.NETWORK_ACCESS_ALLOWED and len(unique_urls) > 1:
|
||||||
|
worker_count = min(URL_CHECK_MAX_WORKERS, len(unique_urls))
|
||||||
|
with ThreadPoolExecutor(max_workers=worker_count) as executor:
|
||||||
|
url_matches = dict(
|
||||||
|
zip(unique_urls, executor.map(self.check_url, unique_urls))
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
url_matches = {url: self.check_url(url) for url in unique_urls}
|
||||||
|
|
||||||
|
return [
|
||||||
|
next(
|
||||||
|
(url_matches[url] for url in urls if url_matches[url] is not None),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
for urls in batches
|
||||||
|
]
|
||||||
|
|
||||||
def check_process(self, process: str) -> Optional[IndicatorMatch]:
|
def check_process(self, process: str) -> Optional[IndicatorMatch]:
|
||||||
"""Check the provided process name against the list of process
|
"""Check the provided process name against the list of process
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from .module_types import (
|
|||||||
ModuleResults,
|
ModuleResults,
|
||||||
ModuleSerializedResult,
|
ModuleSerializedResult,
|
||||||
ModuleTimeline,
|
ModuleTimeline,
|
||||||
|
URLResult,
|
||||||
)
|
)
|
||||||
from .utils import CustomJSONEncoder, exec_or_profile
|
from .utils import CustomJSONEncoder, exec_or_profile
|
||||||
|
|
||||||
@@ -82,6 +83,7 @@ class MVTModule:
|
|||||||
|
|
||||||
self.results: ModuleResults = results if results is not None else []
|
self.results: ModuleResults = results if results is not None else []
|
||||||
self.timeline: ModuleTimeline = []
|
self.timeline: ModuleTimeline = []
|
||||||
|
self.url_results: list[URLResult] = []
|
||||||
self.dependency_modules: Dict[type["MVTModule"], "MVTModule"] = {}
|
self.dependency_modules: Dict[type["MVTModule"], "MVTModule"] = {}
|
||||||
|
|
||||||
def get_dependency_results(
|
def get_dependency_results(
|
||||||
@@ -110,6 +112,23 @@ class MVTModule:
|
|||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def collect_url_results(self) -> None:
|
||||||
|
"""Collect URL records exposed by this module."""
|
||||||
|
|
||||||
|
def add_url_result(self, url: str, timestamp: Optional[str], source: str) -> None:
|
||||||
|
expanded_url = None
|
||||||
|
if self.indicators:
|
||||||
|
expanded_url = self.indicators.get_expanded_url(url)
|
||||||
|
|
||||||
|
self.url_results.append(
|
||||||
|
{
|
||||||
|
"url": url,
|
||||||
|
"expanded_url": expanded_url,
|
||||||
|
"timestamp": timestamp,
|
||||||
|
"source": source,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def save_to_json(self) -> None:
|
def save_to_json(self) -> None:
|
||||||
if not self.results_path:
|
if not self.results_path:
|
||||||
return
|
return
|
||||||
@@ -249,6 +268,15 @@ def run_module(module: MVTModule) -> None:
|
|||||||
"The %s module produced no detections!", module.__class__.__name__
|
"The %s module produced no detections!", module.__class__.__name__
|
||||||
)
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
module.collect_url_results()
|
||||||
|
except Exception as exc:
|
||||||
|
module.log.exception(
|
||||||
|
"Error when collecting URLs from module %s: %s",
|
||||||
|
module.__class__.__name__,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
module.to_timeline()
|
module.to_timeline()
|
||||||
except NotImplementedError:
|
except NotImplementedError:
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import inspect
|
import inspect
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -15,6 +16,7 @@ from typing import Iterable, Optional
|
|||||||
from .module import MVTModule
|
from .module import MVTModule
|
||||||
|
|
||||||
MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES"
|
MVT_CUSTOM_MODULES_ENV = "MVT_CUSTOM_MODULES"
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class CustomModuleLoadError(Exception):
|
class CustomModuleLoadError(Exception):
|
||||||
@@ -122,6 +124,11 @@ def module_supports_command(
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
supported_commands = getattr(module_class, "supported_commands", None)
|
supported_commands = getattr(module_class, "supported_commands", None)
|
||||||
if not supported_commands:
|
if not supported_commands:
|
||||||
return True
|
log.warning(
|
||||||
|
"Custom module %s has no supported_commands and will not be run. "
|
||||||
|
"Declare the platform/command pairs it supports.",
|
||||||
|
module_class.__name__,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
return (platform, command) in {tuple(entry) for entry in supported_commands}
|
return (platform, command) in {tuple(entry) for entry in supported_commands}
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Dict, List, Union
|
from typing import Any, Dict, List, Optional, TypedDict, Union
|
||||||
|
|
||||||
|
|
||||||
# ModuleAtomicResult is a flexible dictionary that can contain any data.
|
# ModuleAtomicResult is a flexible dictionary that can contain any data.
|
||||||
@@ -22,6 +22,13 @@ ModuleAtomicResult = Dict[str, Any]
|
|||||||
ModuleResults = Any
|
ModuleResults = Any
|
||||||
|
|
||||||
|
|
||||||
|
class URLResult(TypedDict):
|
||||||
|
url: str
|
||||||
|
expanded_url: Optional[str]
|
||||||
|
timestamp: Optional[str]
|
||||||
|
source: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ModuleAtomicTimeline:
|
class ModuleAtomicTimeline:
|
||||||
timestamp: str
|
timestamp: str
|
||||||
|
|||||||
@@ -3,4 +3,4 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
MVT_VERSION = "2026.5.12"
|
MVT_VERSION = "2026.7.29"
|
||||||
@@ -8,6 +8,12 @@ import logging
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
import click
|
import click
|
||||||
|
from mvt.common.cli_plugins import (
|
||||||
|
IOS_CLI_PLUGIN_GROUP,
|
||||||
|
MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||||
|
load_cli_commands_option,
|
||||||
|
register_cli_plugins,
|
||||||
|
)
|
||||||
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||||
from mvt.common.completion import (
|
from mvt.common.completion import (
|
||||||
SUPPORTED_SHELLS,
|
SUPPORTED_SHELLS,
|
||||||
@@ -42,6 +48,7 @@ from mvt.common.help import (
|
|||||||
HELP_MSG_CHECK_IOCS,
|
HELP_MSG_CHECK_IOCS,
|
||||||
HELP_MSG_STIX2,
|
HELP_MSG_STIX2,
|
||||||
HELP_MSG_CHECK_IOS_BACKUP,
|
HELP_MSG_CHECK_IOS_BACKUP,
|
||||||
|
HELP_MSG_CHECK_SYSDIAGNOSE,
|
||||||
HELP_MSG_DISABLE_UPDATE_CHECK,
|
HELP_MSG_DISABLE_UPDATE_CHECK,
|
||||||
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
HELP_MSG_DISABLE_INDICATOR_UPDATE_CHECK,
|
||||||
HELP_MSG_COMPLETION,
|
HELP_MSG_COMPLETION,
|
||||||
@@ -50,6 +57,7 @@ from mvt.common.module_loader import CustomModuleLoadError, load_custom_modules
|
|||||||
from mvt.common.password import prompt_password
|
from mvt.common.password import prompt_password
|
||||||
from .cmd_check_backup import CmdIOSCheckBackup
|
from .cmd_check_backup import CmdIOSCheckBackup
|
||||||
from .cmd_check_fs import CmdIOSCheckFS
|
from .cmd_check_fs import CmdIOSCheckFS
|
||||||
|
from .cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
|
||||||
from .decrypt import DecryptBackup
|
from .decrypt import DecryptBackup
|
||||||
from .modules.backup import BACKUP_MODULES
|
from .modules.backup import BACKUP_MODULES
|
||||||
from .modules.fs import FS_MODULES
|
from .modules.fs import FS_MODULES
|
||||||
@@ -84,6 +92,7 @@ def _load_custom_modules(load_module):
|
|||||||
# Main
|
# Main
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@click.group(invoke_without_command=False)
|
@click.group(invoke_without_command=False)
|
||||||
|
@load_cli_commands_option
|
||||||
@click.option(
|
@click.option(
|
||||||
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
"--disable-update-check", is_flag=True, help=HELP_MSG_DISABLE_UPDATE_CHECK
|
||||||
)
|
)
|
||||||
@@ -398,6 +407,77 @@ def check_fs(
|
|||||||
cmd.show_support_message()
|
cmd.show_support_message()
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# Command: check-sysdiagnose
|
||||||
|
# ==============================================================================
|
||||||
|
@cli.command(
|
||||||
|
"check-sysdiagnose",
|
||||||
|
context_settings=CONTEXT_SETTINGS,
|
||||||
|
help=HELP_MSG_CHECK_SYSDIAGNOSE,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--iocs",
|
||||||
|
"-i",
|
||||||
|
type=click.Path(exists=True),
|
||||||
|
multiple=True,
|
||||||
|
default=[],
|
||||||
|
help=HELP_MSG_IOC,
|
||||||
|
)
|
||||||
|
@click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT)
|
||||||
|
@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
|
||||||
|
@click.option("--module", "-m", help=HELP_MSG_MODULE)
|
||||||
|
@click.option(
|
||||||
|
"--load-module",
|
||||||
|
type=click.Path(exists=True),
|
||||||
|
multiple=True,
|
||||||
|
default=[],
|
||||||
|
help=HELP_MSG_LOAD_MODULE,
|
||||||
|
)
|
||||||
|
@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
|
||||||
|
@click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
|
||||||
|
@click.argument("SYSDIAGNOSE_PATH", type=click.Path(exists=True))
|
||||||
|
@click.pass_context
|
||||||
|
def check_sysdiagnose(
|
||||||
|
ctx,
|
||||||
|
iocs,
|
||||||
|
output,
|
||||||
|
list_modules,
|
||||||
|
module,
|
||||||
|
load_module,
|
||||||
|
hashes,
|
||||||
|
verbose,
|
||||||
|
sysdiagnose_path,
|
||||||
|
):
|
||||||
|
set_verbose_logging(verbose)
|
||||||
|
custom_modules = _load_custom_modules(load_module)
|
||||||
|
cmd = CmdIOSCheckSysdiagnose(
|
||||||
|
target_path=sysdiagnose_path,
|
||||||
|
results_path=output,
|
||||||
|
ioc_files=iocs,
|
||||||
|
module_name=module,
|
||||||
|
hashes=hashes,
|
||||||
|
disable_version_check=_get_disable_flags(ctx)[0],
|
||||||
|
disable_indicator_check=_get_disable_flags(ctx)[1],
|
||||||
|
custom_modules=custom_modules,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not cmd._available_modules():
|
||||||
|
raise click.ClickException(
|
||||||
|
"No custom modules support mvt-ios check-sysdiagnose. "
|
||||||
|
"Load a module that declares supported_commands = "
|
||||||
|
"((\"ios\", \"check-sysdiagnose\"),)."
|
||||||
|
)
|
||||||
|
|
||||||
|
if list_modules:
|
||||||
|
cmd.list_modules()
|
||||||
|
return
|
||||||
|
|
||||||
|
log.info("Checking iOS sysdiagnose at path: %s", sysdiagnose_path)
|
||||||
|
cmd.run()
|
||||||
|
cmd.show_alerts_brief()
|
||||||
|
cmd.show_support_message()
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# Command: check-iocs
|
# Command: check-iocs
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@@ -450,3 +530,10 @@ def check_iocs(ctx, iocs, list_modules, module, load_module, folder):
|
|||||||
def download_iocs():
|
def download_iocs():
|
||||||
ioc_updates = IndicatorsUpdates()
|
ioc_updates = IndicatorsUpdates()
|
||||||
ioc_updates.update()
|
ioc_updates.update()
|
||||||
|
|
||||||
|
|
||||||
|
register_cli_plugins(
|
||||||
|
cli,
|
||||||
|
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
||||||
|
environment_variable=MVT_IOS_CUSTOM_COMMANDS_ENV,
|
||||||
|
)
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import tarfile
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from mvt.common.command import Command
|
||||||
|
from mvt.common.indicators import Indicators
|
||||||
|
from mvt.common.module import MVTModule
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class CmdIOSCheckSysdiagnose(Command):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
target_path: Optional[str] = None,
|
||||||
|
results_path: Optional[str] = None,
|
||||||
|
ioc_files: Optional[list] = None,
|
||||||
|
iocs: Optional[Indicators] = None,
|
||||||
|
module_name: Optional[str] = None,
|
||||||
|
serial: Optional[str] = None,
|
||||||
|
module_options: Optional[dict] = None,
|
||||||
|
hashes: bool = False,
|
||||||
|
sub_command: bool = False,
|
||||||
|
disable_version_check: bool = False,
|
||||||
|
disable_indicator_check: bool = False,
|
||||||
|
custom_modules: Optional[list[type[MVTModule]]] = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
target_path=target_path,
|
||||||
|
results_path=results_path,
|
||||||
|
ioc_files=ioc_files,
|
||||||
|
iocs=iocs,
|
||||||
|
module_name=module_name,
|
||||||
|
serial=serial,
|
||||||
|
module_options=module_options,
|
||||||
|
hashes=hashes,
|
||||||
|
sub_command=sub_command,
|
||||||
|
log=log,
|
||||||
|
disable_version_check=disable_version_check,
|
||||||
|
disable_indicator_check=disable_indicator_check,
|
||||||
|
custom_modules=custom_modules,
|
||||||
|
)
|
||||||
|
self.platform = "ios"
|
||||||
|
self.name = "check-sysdiagnose"
|
||||||
|
self.sysdiagnose_format: Optional[str] = None
|
||||||
|
self.sysdiagnose_archive: Optional[tarfile.TarFile] = None
|
||||||
|
self.sysdiagnose_files: list[str] = []
|
||||||
|
self.ips_files: list[dict[str, Any]] = []
|
||||||
|
self.temp_sysdiagnose_dir: Optional[TemporaryDirectory[str]] = None
|
||||||
|
self.extracted_sysdiagnose_path: Optional[str] = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_bugtype_header(data: bytes) -> Optional[int]:
|
||||||
|
try:
|
||||||
|
header = json.loads(data.split(b"\n", 1)[0].decode("utf-8"))
|
||||||
|
return int(header["bug_type"])
|
||||||
|
except (json.JSONDecodeError, KeyError, UnicodeDecodeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _add_ips_file(self, file_path: str, data: bytes) -> None:
|
||||||
|
bug_type = self._parse_bugtype_header(data)
|
||||||
|
if bug_type is not None:
|
||||||
|
self.ips_files.append({"file_path": file_path, "bug_type": bug_type})
|
||||||
|
|
||||||
|
def init(self) -> None:
|
||||||
|
if not self.target_path:
|
||||||
|
raise ValueError("A sysdiagnose path is required")
|
||||||
|
|
||||||
|
if os.path.isdir(self.target_path):
|
||||||
|
self.sysdiagnose_format = "dir"
|
||||||
|
parent_path = Path(self.target_path).absolute().parent
|
||||||
|
for root, _, filenames in os.walk(self.target_path):
|
||||||
|
for filename in filenames:
|
||||||
|
absolute_path = os.path.join(root, filename)
|
||||||
|
file_path = os.path.relpath(absolute_path, parent_path)
|
||||||
|
self.sysdiagnose_files.append(file_path)
|
||||||
|
if filename.endswith(".ips"):
|
||||||
|
with open(absolute_path, "rb") as handle:
|
||||||
|
self._add_ips_file(absolute_path, handle.read())
|
||||||
|
return
|
||||||
|
|
||||||
|
if not os.path.isfile(self.target_path):
|
||||||
|
raise ValueError(f"Sysdiagnose path does not exist: {self.target_path}")
|
||||||
|
|
||||||
|
self.log.info("Parsing sysdiagnose archive. This might take a while...")
|
||||||
|
self.sysdiagnose_format = "tar"
|
||||||
|
self.sysdiagnose_archive = tarfile.open(self.target_path, "r:gz")
|
||||||
|
self._extract_sysdiagnose_archive()
|
||||||
|
|
||||||
|
def _extract_sysdiagnose_archive(self) -> None:
|
||||||
|
archive = self.sysdiagnose_archive
|
||||||
|
if archive is None:
|
||||||
|
raise RuntimeError("Sysdiagnose archive has not been initialized")
|
||||||
|
|
||||||
|
self.temp_sysdiagnose_dir = TemporaryDirectory()
|
||||||
|
extraction_root = Path(self.temp_sysdiagnose_dir.name).resolve()
|
||||||
|
archive_roots = set()
|
||||||
|
|
||||||
|
for member in archive:
|
||||||
|
member_path = PurePosixPath(member.name.replace("\\", "/"))
|
||||||
|
if member_path.is_absolute() or ".." in member_path.parts:
|
||||||
|
self.log.warning("Skipping unsafe sysdiagnose path %r", member.name)
|
||||||
|
continue
|
||||||
|
|
||||||
|
destination = extraction_root.joinpath(*member_path.parts).resolve()
|
||||||
|
if not destination.is_relative_to(extraction_root):
|
||||||
|
self.log.warning("Skipping unsafe sysdiagnose path %r", member.name)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not member_path.parts:
|
||||||
|
continue
|
||||||
|
archive_roots.add(member_path.parts[0])
|
||||||
|
|
||||||
|
if member.isdir():
|
||||||
|
destination.mkdir(parents=True, exist_ok=True)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Modules only need directories and regular files. Do not materialize
|
||||||
|
# links or device nodes from an untrusted sysdiagnose archive.
|
||||||
|
if not member.isfile():
|
||||||
|
self.log.warning("Skipping unsafe sysdiagnose member %r", member.name)
|
||||||
|
continue
|
||||||
|
|
||||||
|
normalized_name = member_path.as_posix()
|
||||||
|
self.sysdiagnose_files.append(normalized_name)
|
||||||
|
|
||||||
|
source = archive.extractfile(member)
|
||||||
|
if source is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with source, destination.open("wb") as output:
|
||||||
|
shutil.copyfileobj(source, output)
|
||||||
|
|
||||||
|
if normalized_name.endswith(".ips"):
|
||||||
|
self._add_ips_file(str(destination), destination.read_bytes())
|
||||||
|
|
||||||
|
if len(archive_roots) != 1:
|
||||||
|
raise ValueError("Sysdiagnose archive must contain one top-level directory")
|
||||||
|
|
||||||
|
self.extracted_sysdiagnose_path = str(extraction_root / archive_roots.pop())
|
||||||
|
|
||||||
|
def module_init(self, module) -> None:
|
||||||
|
module.ips_files = self.ips_files
|
||||||
|
if self.sysdiagnose_format == "tar":
|
||||||
|
if self.extracted_sysdiagnose_path is None:
|
||||||
|
raise RuntimeError("Sysdiagnose archive has not been extracted")
|
||||||
|
module.from_sysdiagnose_folder(
|
||||||
|
self.extracted_sysdiagnose_path, self.sysdiagnose_files
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if self.sysdiagnose_format == "dir" and self.target_path:
|
||||||
|
module.from_sysdiagnose_folder(self.target_path, self.sysdiagnose_files)
|
||||||
|
return
|
||||||
|
raise RuntimeError("Sysdiagnose input has not been initialized")
|
||||||
|
|
||||||
|
def finish(self) -> None:
|
||||||
|
if self.sysdiagnose_archive is not None:
|
||||||
|
self.sysdiagnose_archive.close()
|
||||||
|
self.sysdiagnose_archive = None
|
||||||
|
if self.temp_sysdiagnose_dir is not None:
|
||||||
|
self.temp_sysdiagnose_dir.cleanup()
|
||||||
|
self.temp_sysdiagnose_dir = None
|
||||||
@@ -1216,6 +1216,14 @@
|
|||||||
"version": "18.7.9",
|
"version": "18.7.9",
|
||||||
"build": "22H355"
|
"build": "22H355"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"version": "18.7.10",
|
||||||
|
"build": "22H373"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "18.7.10",
|
||||||
|
"build": "22H374"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"version": "26",
|
"version": "26",
|
||||||
"build": "23A341"
|
"build": "23A341"
|
||||||
@@ -1255,5 +1263,13 @@
|
|||||||
{
|
{
|
||||||
"version": "26.5",
|
"version": "26.5",
|
||||||
"build": "23F77"
|
"build": "23F77"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "26.6.1",
|
||||||
|
"build": "23G82"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "26.6.1",
|
||||||
|
"build": "23G83"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
+63
-20
@@ -9,6 +9,7 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Iterator, Optional, Union
|
from typing import Iterator, Optional, Union
|
||||||
|
|
||||||
@@ -20,6 +21,20 @@ from mvt.common.module import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TemporarySQLiteConnection(sqlite3.Connection):
|
||||||
|
"""SQLite connection that owns a temporary copy of a database."""
|
||||||
|
|
||||||
|
temporary_directory: Optional[tempfile.TemporaryDirectory] = None
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
try:
|
||||||
|
super().close()
|
||||||
|
finally:
|
||||||
|
if self.temporary_directory:
|
||||||
|
self.temporary_directory.cleanup()
|
||||||
|
self.temporary_directory = None
|
||||||
|
|
||||||
|
|
||||||
class IOSExtraction(MVTModule):
|
class IOSExtraction(MVTModule):
|
||||||
"""This class provides a base for all iOS filesystem/backup extraction
|
"""This class provides a base for all iOS filesystem/backup extraction
|
||||||
modules."""
|
modules."""
|
||||||
@@ -44,6 +59,8 @@ class IOSExtraction(MVTModule):
|
|||||||
|
|
||||||
self.is_backup = False
|
self.is_backup = False
|
||||||
self.is_fs_dump = False
|
self.is_fs_dump = False
|
||||||
|
self._recovered_sqlite_paths: dict[str, str] = {}
|
||||||
|
self._sqlite_temp_directories: list[tempfile.TemporaryDirectory] = []
|
||||||
|
|
||||||
def _recover_sqlite_db_if_needed(
|
def _recover_sqlite_db_if_needed(
|
||||||
self, file_path: str, forced: bool = False
|
self, file_path: str, forced: bool = False
|
||||||
@@ -53,17 +70,12 @@ class IOSExtraction(MVTModule):
|
|||||||
:param file_path: Path to the malformed database file.
|
:param file_path: Path to the malformed database file.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
# SQLite's immutable mode cannot open databases with active WAL files.
|
|
||||||
if not forced:
|
if not forced:
|
||||||
# If the database is open, do not use immutable
|
conn = self._open_sqlite_db(file_path)
|
||||||
if os.path.isfile(file_path + "-shm"):
|
|
||||||
conn = sqlite3.connect(file_path)
|
|
||||||
else:
|
|
||||||
conn = self._open_sqlite_db(file_path)
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
recover = False
|
||||||
try:
|
try:
|
||||||
recover = False
|
|
||||||
cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
||||||
except sqlite3.DatabaseError as exc:
|
except sqlite3.DatabaseError as exc:
|
||||||
if "database disk image is malformed" in str(exc):
|
if "database disk image is malformed" in str(exc):
|
||||||
@@ -82,27 +94,54 @@ class IOSExtraction(MVTModule):
|
|||||||
raise DatabaseCorruptedError(
|
raise DatabaseCorruptedError(
|
||||||
"failed to recover without sqlite3 binary: please install sqlite3!"
|
"failed to recover without sqlite3 binary: please install sqlite3!"
|
||||||
)
|
)
|
||||||
if '"' in file_path:
|
temporary_directory = tempfile.TemporaryDirectory(prefix="mvt_sqlite_recover_")
|
||||||
raise DatabaseCorruptedError(
|
temporary_path = Path(temporary_directory.name)
|
||||||
f"database at path '{file_path}' is corrupted. unable to "
|
source_path = temporary_path / "source.db"
|
||||||
'recover because it has a quotation mark (") in its name'
|
recovered_path = temporary_path / "recovered.db"
|
||||||
)
|
shutil.copy2(file_path, source_path)
|
||||||
|
for suffix in ("-wal", "-shm"):
|
||||||
bak_path = f"{file_path}.bak"
|
sidecar = Path(file_path + suffix)
|
||||||
shutil.move(file_path, bak_path)
|
if sidecar.is_file():
|
||||||
|
shutil.copy2(sidecar, Path(str(source_path) + suffix))
|
||||||
|
|
||||||
ret = subprocess.call(
|
ret = subprocess.call(
|
||||||
["sqlite3", bak_path, f'.clone "{file_path}"'],
|
["sqlite3", str(source_path), f'.clone "{recovered_path}"'],
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.PIPE,
|
stderr=subprocess.PIPE,
|
||||||
)
|
)
|
||||||
if ret != 0:
|
if ret != 0:
|
||||||
|
temporary_directory.cleanup()
|
||||||
raise DatabaseCorruptedError("failed to recover database")
|
raise DatabaseCorruptedError("failed to recover database")
|
||||||
|
|
||||||
|
self._sqlite_temp_directories.append(temporary_directory)
|
||||||
|
self._recovered_sqlite_paths[file_path] = str(recovered_path)
|
||||||
self.log.info("Database at path %s recovered successfully!", file_path)
|
self.log.info("Database at path %s recovered successfully!", file_path)
|
||||||
|
|
||||||
def _open_sqlite_db(self, file_path: str) -> sqlite3.Connection:
|
def _open_sqlite_db(self, file_path: str) -> sqlite3.Connection:
|
||||||
return sqlite3.connect(f"file:{file_path}?immutable=1", uri=True)
|
database_path = self._recovered_sqlite_paths.get(file_path, file_path)
|
||||||
|
if not os.path.isfile(database_path + "-wal"):
|
||||||
|
uri = Path(database_path).resolve().as_uri() + "?mode=ro&immutable=1"
|
||||||
|
return sqlite3.connect(uri, uri=True)
|
||||||
|
|
||||||
|
temporary_directory = tempfile.TemporaryDirectory(prefix="mvt_sqlite_")
|
||||||
|
temporary_path = Path(temporary_directory.name) / Path(database_path).name
|
||||||
|
shutil.copy2(database_path, temporary_path)
|
||||||
|
for suffix in ("-wal", "-shm"):
|
||||||
|
sidecar = Path(database_path + suffix)
|
||||||
|
if sidecar.is_file():
|
||||||
|
shutil.copy2(sidecar, Path(str(temporary_path) + suffix))
|
||||||
|
|
||||||
|
try:
|
||||||
|
conn = sqlite3.connect(
|
||||||
|
temporary_path.resolve().as_uri() + "?mode=ro",
|
||||||
|
uri=True,
|
||||||
|
factory=TemporarySQLiteConnection,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
temporary_directory.cleanup()
|
||||||
|
raise
|
||||||
|
conn.temporary_directory = temporary_directory
|
||||||
|
return conn
|
||||||
|
|
||||||
def _get_backup_files_from_manifest(
|
def _get_backup_files_from_manifest(
|
||||||
self, relative_path: Optional[str] = None, domain: Optional[str] = None
|
self, relative_path: Optional[str] = None, domain: Optional[str] = None
|
||||||
@@ -166,7 +205,11 @@ class IOSExtraction(MVTModule):
|
|||||||
if not self.target_path:
|
if not self.target_path:
|
||||||
return None
|
return None
|
||||||
file_path = os.path.join(self.target_path, file_id[0:2], file_id)
|
file_path = os.path.join(self.target_path, file_id[0:2], file_id)
|
||||||
if not Path(file_path).resolve().is_relative_to(Path(self.target_path).resolve()):
|
if (
|
||||||
|
not Path(file_path)
|
||||||
|
.resolve()
|
||||||
|
.is_relative_to(Path(self.target_path).resolve())
|
||||||
|
):
|
||||||
return None
|
return None
|
||||||
if os.path.exists(file_path):
|
if os.path.exists(file_path):
|
||||||
return file_path
|
return file_path
|
||||||
@@ -197,9 +240,9 @@ class IOSExtraction(MVTModule):
|
|||||||
:param backup_ids: Default value = None)
|
:param backup_ids: Default value = None)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file_path: Optional[str] = None
|
file_path: Optional[str] = self.file_path
|
||||||
# First we check if the was an explicit file path specified.
|
# First we check if the was an explicit file path specified.
|
||||||
if not self.file_path:
|
if not file_path:
|
||||||
# Type narrowing: we know self.file_path is None here, work with local file_path
|
# Type narrowing: we know self.file_path is None here, work with local file_path
|
||||||
# If not, we first try with backups.
|
# If not, we first try with backups.
|
||||||
# We construct the path to the file according to the iTunes backup
|
# We construct the path to the file according to the iTunes backup
|
||||||
|
|||||||
@@ -134,6 +134,8 @@ class Analytics(IOSExtraction):
|
|||||||
isodate = ""
|
isodate = ""
|
||||||
data = plistlib.loads(row[1])
|
data = plistlib.loads(row[1])
|
||||||
data["isodate"] = isodate
|
data["isodate"] = isodate
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
data["artifact"] = artifact
|
data["artifact"] = artifact
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from .tcc import TCC
|
|||||||
from .webkit_resource_load_statistics import WebkitResourceLoadStatistics
|
from .webkit_resource_load_statistics import WebkitResourceLoadStatistics
|
||||||
from .webkit_session_resource_log import WebkitSessionResourceLog
|
from .webkit_session_resource_log import WebkitSessionResourceLog
|
||||||
from .whatsapp import Whatsapp
|
from .whatsapp import Whatsapp
|
||||||
|
from .whatsapp_contacts import WhatsappContacts
|
||||||
|
|
||||||
MIXED_MODULES = [
|
MIXED_MODULES = [
|
||||||
Calls,
|
Calls,
|
||||||
@@ -47,6 +48,7 @@ MIXED_MODULES = [
|
|||||||
WebkitResourceLoadStatistics,
|
WebkitResourceLoadStatistics,
|
||||||
WebkitSessionResourceLog,
|
WebkitSessionResourceLog,
|
||||||
Whatsapp,
|
Whatsapp,
|
||||||
|
WhatsappContacts,
|
||||||
Shortcuts,
|
Shortcuts,
|
||||||
Applications,
|
Applications,
|
||||||
Calendar,
|
Calendar,
|
||||||
|
|||||||
@@ -73,7 +73,9 @@ class Calls(IOSExtraction):
|
|||||||
"duration": row[1],
|
"duration": row[1],
|
||||||
"location": row[2],
|
"location": row[2],
|
||||||
"number": (
|
"number": (
|
||||||
row[3].decode("utf-8") if row[3] and row[3] is bytes else row[3]
|
row[3].decode("utf-8")
|
||||||
|
if isinstance(row[3], bytes)
|
||||||
|
else row[3]
|
||||||
),
|
),
|
||||||
"provider": row[4],
|
"provider": row[4],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,9 +3,11 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from typing import Optional
|
from typing import Optional, Tuple
|
||||||
|
|
||||||
from mvt.common.module_types import (
|
from mvt.common.module_types import (
|
||||||
ModuleAtomicResult,
|
ModuleAtomicResult,
|
||||||
@@ -15,6 +17,7 @@ from mvt.common.module_types import (
|
|||||||
from mvt.common.utils import convert_mactime_to_iso
|
from mvt.common.utils import convert_mactime_to_iso
|
||||||
|
|
||||||
from ..base import IOSExtraction
|
from ..base import IOSExtraction
|
||||||
|
from .whatsapp_contacts import WhatsappContacts
|
||||||
|
|
||||||
INTERACTIONC_BACKUP_IDS = [
|
INTERACTIONC_BACKUP_IDS = [
|
||||||
"1f5a521220a3ad80ebfdc196978df8e7a2e49dee",
|
"1f5a521220a3ad80ebfdc196978df8e7a2e49dee",
|
||||||
@@ -22,6 +25,49 @@ INTERACTIONC_BACKUP_IDS = [
|
|||||||
INTERACTIONC_ROOT_PATHS = [
|
INTERACTIONC_ROOT_PATHS = [
|
||||||
"private/var/mobile/Library/CoreDuet/People/interactionC.db",
|
"private/var/mobile/Library/CoreDuet/People/interactionC.db",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# The interaction record's creation date normally trails its start date by
|
||||||
|
# milliseconds: emitting it as a timeline event only duplicates the start
|
||||||
|
# date event. A large divergence, however, indicates the record was
|
||||||
|
# backfilled (sync, restore, or tampering) and is worth surfacing.
|
||||||
|
CREATION_DATE_DIVERGENCE_THRESHOLD = 3600.0
|
||||||
|
|
||||||
|
# Per-contact aggregate dates from ZCONTACTS are repeated on every
|
||||||
|
# interaction row of the same contact. They are serialized with a
|
||||||
|
# contact-centric data string so that timeline de-duplication collapses
|
||||||
|
# them into one event per contact.
|
||||||
|
CONTACT_EVENT_TEMPLATES = {
|
||||||
|
"contacts_creation_date": "Contact {party} first recorded in interactionC",
|
||||||
|
"first_incoming_sender_date": "First incoming interaction from {party}",
|
||||||
|
"last_incoming_sender_date": "Last incoming interaction from {party}",
|
||||||
|
"first_incoming_recipient_date": (
|
||||||
|
"First incoming interaction where {party} was a recipient"
|
||||||
|
),
|
||||||
|
"last_incoming_recipient_date": (
|
||||||
|
"Last incoming interaction where {party} was a recipient"
|
||||||
|
),
|
||||||
|
"first_outgoing_recipient_date": (
|
||||||
|
"First outgoing interaction to {party}"
|
||||||
|
),
|
||||||
|
"last_outgoing_recipient_date": (
|
||||||
|
"Last outgoing interaction to {party}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_iso(timestamp) -> Optional[datetime.datetime]:
|
||||||
|
try:
|
||||||
|
return datetime.datetime.strptime(
|
||||||
|
timestamp, "%Y-%m-%d %H:%M:%S.%f"
|
||||||
|
)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _describe_delta(seconds: float) -> str:
|
||||||
|
if seconds >= 86400:
|
||||||
|
return f"{seconds / 86400:.0f} days"
|
||||||
|
return f"{seconds / 3600:.0f} hours"
|
||||||
# Taken from APOLLO
|
# Taken from APOLLO
|
||||||
# https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt
|
# https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt
|
||||||
QUERIES = [
|
QUERIES = [
|
||||||
@@ -34,7 +80,7 @@ QUERIES = [
|
|||||||
CASE ZINTERACTIONS.ZDIRECTION
|
CASE ZINTERACTIONS.ZDIRECTION
|
||||||
WHEN '0' THEN 'INCOMING'
|
WHEN '0' THEN 'INCOMING'
|
||||||
WHEN '1' THEN 'OUTGOING'
|
WHEN '1' THEN 'OUTGOING'
|
||||||
END 'DIRECTION' AS "direction",
|
END AS "direction",
|
||||||
ZCONTACTS.ZDISPLAYNAME AS "sender_display_name",
|
ZCONTACTS.ZDISPLAYNAME AS "sender_display_name",
|
||||||
ZCONTACTS.ZIDENTIFIER AS "sender_identifier",
|
ZCONTACTS.ZIDENTIFIER AS "sender_identifier",
|
||||||
ZCONTACTS.ZPERSONID AS "sender_personid",
|
ZCONTACTS.ZPERSONID AS "sender_personid",
|
||||||
@@ -89,7 +135,7 @@ QUERIES = [
|
|||||||
CASE ZINTERACTIONS.ZDIRECTION
|
CASE ZINTERACTIONS.ZDIRECTION
|
||||||
WHEN '0' THEN 'INCOMING'
|
WHEN '0' THEN 'INCOMING'
|
||||||
WHEN '1' THEN 'OUTGOING'
|
WHEN '1' THEN 'OUTGOING'
|
||||||
END 'DIRECTION' AS "direction",
|
END AS "direction",
|
||||||
ZCONTACTS.ZDISPLAYNAME AS "sender_display_name",
|
ZCONTACTS.ZDISPLAYNAME AS "sender_display_name",
|
||||||
ZCONTACTS.ZIDENTIFIER AS "sender_identifier",
|
ZCONTACTS.ZIDENTIFIER AS "sender_identifier",
|
||||||
ZCONTACTS.ZPERSONID AS "sender_personid",
|
ZCONTACTS.ZPERSONID AS "sender_personid",
|
||||||
@@ -117,7 +163,7 @@ QUERIES = [
|
|||||||
CASE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE
|
CASE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE
|
||||||
WHEN '0' THEN '0'
|
WHEN '0' THEN '0'
|
||||||
ELSE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE
|
ELSE ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE
|
||||||
END 'LAST INCOMING RECIPIENT DATE' AS "last_incoming_recipient_date",
|
END AS "last_incoming_recipient_date",
|
||||||
ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE AS "last_outgoing_recipient_date",
|
ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE AS "last_outgoing_recipient_date",
|
||||||
ZCONTACTS.ZCUSTOMIDENTIFIER AS "custom_id",
|
ZCONTACTS.ZCUSTOMIDENTIFIER AS "custom_id",
|
||||||
ZINTERACTIONS.ZCONTENTURL AS "interaction_content_url",
|
ZINTERACTIONS.ZCONTENTURL AS "interaction_content_url",
|
||||||
@@ -218,9 +264,16 @@ QUERIES = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
WHATSAPP_BUNDLE_ID = "net.whatsapp.WhatsApp"
|
||||||
|
|
||||||
|
|
||||||
class InteractionC(IOSExtraction):
|
class InteractionC(IOSExtraction):
|
||||||
"""This module extracts data from InteractionC db."""
|
"""This module extracts data from InteractionC db."""
|
||||||
|
|
||||||
|
# WhatsApp identifies chat peers by LID in interactionC.db, which only the
|
||||||
|
# WhatsApp contacts database can map back to a phone number and name.
|
||||||
|
dependencies = [WhatsappContacts]
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
file_path: Optional[str] = None,
|
file_path: Optional[str] = None,
|
||||||
@@ -252,10 +305,51 @@ class InteractionC(IOSExtraction):
|
|||||||
"last_outgoing_recipient_date",
|
"last_outgoing_recipient_date",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _describe_party(record: ModuleAtomicResult, prefix: str) -> Optional[str]:
|
||||||
|
name = record.get(f"{prefix}_display_name") or record.get(
|
||||||
|
f"{prefix}_resolved_name"
|
||||||
|
)
|
||||||
|
identifier = record.get(f"{prefix}_resolved_phone_number") or record.get(
|
||||||
|
f"{prefix}_identifier"
|
||||||
|
)
|
||||||
|
if name and identifier:
|
||||||
|
# A display name that is just a formatted copy of the phone
|
||||||
|
# number adds no information.
|
||||||
|
name_digits = re.sub(r"\D", "", name)
|
||||||
|
if name_digits and name_digits == re.sub(r"\D", "", identifier):
|
||||||
|
return identifier
|
||||||
|
return f"{name} ({identifier})"
|
||||||
|
return name or identifier or None
|
||||||
|
|
||||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||||
|
sender = self._describe_party(record, "sender")
|
||||||
|
# The chat peer from the domain identifier stands in when the
|
||||||
|
# recipient was not recorded (or the recipient join is unavailable).
|
||||||
|
recipient = self._describe_party(record, "recipient") or self._describe_party(
|
||||||
|
record, "domain"
|
||||||
|
)
|
||||||
|
direction = record.get("direction")
|
||||||
|
if not sender and direction == "OUTGOING":
|
||||||
|
sender = "local user"
|
||||||
|
if not recipient and direction == "INCOMING":
|
||||||
|
recipient = "local user"
|
||||||
|
|
||||||
|
header = f"[{record['bundle_id']}]"
|
||||||
|
if record.get("account"):
|
||||||
|
header += f" {record['account']}"
|
||||||
|
if direction:
|
||||||
|
header += f" {direction}"
|
||||||
|
|
||||||
|
data = f"{header} from {sender or 'unknown'} to {recipient or 'unknown'}"
|
||||||
|
if record.get("group_name"):
|
||||||
|
data += f" (group: {record['group_name']})"
|
||||||
|
if record.get("content"):
|
||||||
|
data += f": {record['content']}"
|
||||||
|
|
||||||
records = []
|
records = []
|
||||||
processed = []
|
processed = []
|
||||||
for timestamp in self.timestamps:
|
for timestamp in ("start_date", "end_date"):
|
||||||
# Check if the record has the current timestamp.
|
# Check if the record has the current timestamp.
|
||||||
if timestamp not in record or not record[timestamp]:
|
if timestamp not in record or not record[timestamp]:
|
||||||
continue
|
continue
|
||||||
@@ -269,16 +363,142 @@ class InteractionC(IOSExtraction):
|
|||||||
"timestamp": record[timestamp],
|
"timestamp": record[timestamp],
|
||||||
"module": self.__class__.__name__,
|
"module": self.__class__.__name__,
|
||||||
"event": timestamp,
|
"event": timestamp,
|
||||||
"data": f"[{record['bundle_id']}] {record['account']} - "
|
"data": data,
|
||||||
f"from {record['sender_display_name']} ({record['sender_identifier']}) "
|
|
||||||
f"to {record.get('recipient_display_name', '')} ({record.get('recipient_identifier', '')}):"
|
|
||||||
f" {record.get('content', '')}",
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
processed.append(record[timestamp])
|
processed.append(record[timestamp])
|
||||||
|
|
||||||
|
creation_event = self._serialize_creation_date(record, data)
|
||||||
|
if creation_event:
|
||||||
|
records.append(creation_event)
|
||||||
|
|
||||||
|
# Contact-level aggregates describe the sender's contact record.
|
||||||
|
party = self._describe_party(record, "sender")
|
||||||
|
if party:
|
||||||
|
for field, template in CONTACT_EVENT_TEMPLATES.items():
|
||||||
|
if not record.get(field):
|
||||||
|
continue
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record[field],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": field,
|
||||||
|
"data": template.format(party=party),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
return records
|
return records
|
||||||
|
|
||||||
|
def _serialize_creation_date(
|
||||||
|
self, record: ModuleAtomicResult, data: str
|
||||||
|
) -> Optional[dict]:
|
||||||
|
"""Serialize the interaction record's creation date only when it
|
||||||
|
diverges from the start date enough to indicate the record was
|
||||||
|
backfilled."""
|
||||||
|
creation = record.get("interactions_creation_date")
|
||||||
|
if not creation:
|
||||||
|
return None
|
||||||
|
|
||||||
|
event = {
|
||||||
|
"timestamp": creation,
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "interactions_creation_date",
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
|
|
||||||
|
start = _parse_iso(record.get("start_date"))
|
||||||
|
creation_parsed = _parse_iso(creation)
|
||||||
|
if not start or not creation_parsed:
|
||||||
|
# Without a start date the creation date is the only anchor.
|
||||||
|
return event
|
||||||
|
|
||||||
|
delta = (creation_parsed - start).total_seconds()
|
||||||
|
if abs(delta) < CREATION_DATE_DIVERGENCE_THRESHOLD:
|
||||||
|
return None
|
||||||
|
|
||||||
|
direction = "after" if delta > 0 else "before"
|
||||||
|
event["data"] = (
|
||||||
|
f"Interaction record created {_describe_delta(abs(delta))} "
|
||||||
|
f"{direction} the event: {data}"
|
||||||
|
)
|
||||||
|
return event
|
||||||
|
|
||||||
|
def _whatsapp_contact_maps(self) -> Tuple[dict, dict]:
|
||||||
|
"""Build LID and phone-digit lookup maps from the WhatsappContacts
|
||||||
|
module results, when available."""
|
||||||
|
by_lid: dict = {}
|
||||||
|
by_phone: dict = {}
|
||||||
|
contacts_module = self.dependency_modules.get(WhatsappContacts)
|
||||||
|
if not contacts_module:
|
||||||
|
return by_lid, by_phone
|
||||||
|
|
||||||
|
for contact in contacts_module.results:
|
||||||
|
name = contact.get("full_name") or contact.get("given_name")
|
||||||
|
phone = contact.get("phone_number")
|
||||||
|
entry = (phone, name)
|
||||||
|
if contact.get("lid"):
|
||||||
|
by_lid[contact["lid"]] = entry
|
||||||
|
if phone:
|
||||||
|
by_phone[re.sub(r"\D", "", phone)] = entry
|
||||||
|
whatsapp_id = contact.get("whatsapp_id")
|
||||||
|
if whatsapp_id and "@" in whatsapp_id:
|
||||||
|
by_phone.setdefault(whatsapp_id.split("@")[0], entry)
|
||||||
|
|
||||||
|
return by_lid, by_phone
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolve_whatsapp_identifier(
|
||||||
|
value, by_lid: dict, by_phone: dict
|
||||||
|
) -> Tuple[Optional[str], Optional[str]]:
|
||||||
|
"""Resolve a WhatsApp identifier (LID, JID or phone number) to a
|
||||||
|
(phone_number, contact_name) tuple."""
|
||||||
|
if not value:
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
value = str(value)
|
||||||
|
if value.endswith("@lid"):
|
||||||
|
return by_lid.get(value, (None, None))
|
||||||
|
if value.endswith("@g.us"):
|
||||||
|
return None, None
|
||||||
|
if value.endswith("@s.whatsapp.net"):
|
||||||
|
digits = value.split("@")[0]
|
||||||
|
phone, name = by_phone.get(digits, (None, None))
|
||||||
|
return phone or f"+{digits}", name
|
||||||
|
if value.startswith("+"):
|
||||||
|
_, name = by_phone.get(re.sub(r"\D", "", value), (None, None))
|
||||||
|
return None, name
|
||||||
|
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
def _postprocess_results(self) -> None:
|
||||||
|
by_lid, by_phone = self._whatsapp_contact_maps()
|
||||||
|
|
||||||
|
for entry in self.results:
|
||||||
|
# The fallback queries return ZDIRECTION raw instead of labelled.
|
||||||
|
if entry.get("direction") in (0, "0"):
|
||||||
|
entry["direction"] = "INCOMING"
|
||||||
|
elif entry.get("direction") in (1, "1"):
|
||||||
|
entry["direction"] = "OUTGOING"
|
||||||
|
|
||||||
|
if entry.get("bundle_id") != WHATSAPP_BUNDLE_ID:
|
||||||
|
continue
|
||||||
|
|
||||||
|
candidates = {
|
||||||
|
"sender": [entry.get("sender_identifier"), entry.get("custom_id")],
|
||||||
|
"recipient": [entry.get("recipient_identifier")],
|
||||||
|
"domain": [entry.get("domain_identifier")],
|
||||||
|
}
|
||||||
|
for prefix, values in candidates.items():
|
||||||
|
phone = name = None
|
||||||
|
for value in values:
|
||||||
|
phone, name = self._resolve_whatsapp_identifier(
|
||||||
|
value, by_lid, by_phone
|
||||||
|
)
|
||||||
|
if phone or name:
|
||||||
|
break
|
||||||
|
entry[f"{prefix}_resolved_phone_number"] = phone
|
||||||
|
entry[f"{prefix}_resolved_name"] = name
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
self._find_ios_database(
|
self._find_ios_database(
|
||||||
backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS
|
backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS
|
||||||
@@ -325,4 +545,6 @@ class InteractionC(IOSExtraction):
|
|||||||
cur.close()
|
cur.close()
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
self._postprocess_results()
|
||||||
|
|
||||||
self.log.info("Extracted a total of %d InteractionC events", len(self.results))
|
self.log.info("Extracted a total of %d InteractionC events", len(self.results))
|
||||||
@@ -19,10 +19,17 @@ from mvt.common.utils import convert_mactime_to_iso, keys_bytes_to_string
|
|||||||
|
|
||||||
from ..base import IOSExtraction
|
from ..base import IOSExtraction
|
||||||
|
|
||||||
SAFARI_BROWSER_STATE_BACKUP_RELPATH = "Library/Safari/BrowserState.db"
|
# Safari profiles (iOS 17 and later) keep their per-profile databases under
|
||||||
|
# Library/Safari/Profiles/<UUID>/, separate from the default profile's.
|
||||||
|
SAFARI_BROWSER_STATE_BACKUP_RELPATHS = [
|
||||||
|
"Library/Safari/BrowserState.db",
|
||||||
|
"Library/Safari/Profiles/*/BrowserState.db",
|
||||||
|
]
|
||||||
SAFARI_BROWSER_STATE_ROOT_PATHS = [
|
SAFARI_BROWSER_STATE_ROOT_PATHS = [
|
||||||
"private/var/mobile/Library/Safari/BrowserState.db",
|
"private/var/mobile/Library/Safari/BrowserState.db",
|
||||||
|
"private/var/mobile/Library/Safari/Profiles/*/BrowserState.db",
|
||||||
"private/var/mobile/Containers/Data/Application/*/Library/Safari/BrowserState.db",
|
"private/var/mobile/Containers/Data/Application/*/Library/Safari/BrowserState.db",
|
||||||
|
"private/var/mobile/Containers/Data/Application/*/Library/Safari/Profiles/*/BrowserState.db",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -174,20 +181,22 @@ class SafariBrowserState(IOSExtraction):
|
|||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
if self.is_backup:
|
if self.is_backup:
|
||||||
for backup_file in self._get_backup_files_from_manifest(
|
for relative_path in SAFARI_BROWSER_STATE_BACKUP_RELPATHS:
|
||||||
relative_path=SAFARI_BROWSER_STATE_BACKUP_RELPATH
|
for backup_file in self._get_backup_files_from_manifest(
|
||||||
):
|
relative_path=relative_path
|
||||||
browserstate_path = self._get_backup_file_from_id(
|
):
|
||||||
backup_file["file_id"]
|
browserstate_path = self._get_backup_file_from_id(
|
||||||
)
|
backup_file["file_id"]
|
||||||
|
)
|
||||||
|
|
||||||
if not browserstate_path:
|
if not browserstate_path:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self.log.info(
|
self.log.info(
|
||||||
"Found Safari browser state database at path: %s", browserstate_path
|
"Found Safari browser state database at path: %s",
|
||||||
)
|
browserstate_path,
|
||||||
self._process_browser_state_db(browserstate_path)
|
)
|
||||||
|
self._process_browser_state_db(browserstate_path)
|
||||||
elif self.is_fs_dump:
|
elif self.is_fs_dump:
|
||||||
for browserstate_path in self._get_fs_files_from_patterns(
|
for browserstate_path in self._get_fs_files_from_patterns(
|
||||||
SAFARI_BROWSER_STATE_ROOT_PATHS
|
SAFARI_BROWSER_STATE_ROOT_PATHS
|
||||||
|
|||||||
@@ -17,10 +17,17 @@ from mvt.common.utils import convert_mactime_to_datetime, convert_mactime_to_iso
|
|||||||
|
|
||||||
from ..base import IOSExtraction
|
from ..base import IOSExtraction
|
||||||
|
|
||||||
SAFARI_HISTORY_BACKUP_RELPATH = "Library/Safari/History.db"
|
# Safari profiles (iOS 17 and later) each keep their own History.db under
|
||||||
|
# Library/Safari/Profiles/<UUID>/, separate from the default profile's database.
|
||||||
|
SAFARI_HISTORY_BACKUP_RELPATHS = [
|
||||||
|
"Library/Safari/History.db",
|
||||||
|
"Library/Safari/Profiles/*/History.db",
|
||||||
|
]
|
||||||
SAFARI_HISTORY_ROOT_PATHS = [
|
SAFARI_HISTORY_ROOT_PATHS = [
|
||||||
"private/var/mobile/Library/Safari/History.db",
|
"private/var/mobile/Library/Safari/History.db",
|
||||||
|
"private/var/mobile/Library/Safari/Profiles/*/History.db",
|
||||||
"private/var/mobile/Containers/Data/Application/*/Library/Safari/History.db",
|
"private/var/mobile/Containers/Data/Application/*/Library/Safari/History.db",
|
||||||
|
"private/var/mobile/Containers/Data/Application/*/Library/Safari/Profiles/*/History.db",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -75,6 +82,9 @@ class SafariHistory(IOSExtraction):
|
|||||||
|
|
||||||
# We loop again through visits in order to find redirect record.
|
# We loop again through visits in order to find redirect record.
|
||||||
for redirect in self.results:
|
for redirect in self.results:
|
||||||
|
if redirect["safari_history_db"] != result["safari_history_db"]:
|
||||||
|
continue
|
||||||
|
|
||||||
if redirect["visit_id"] != result["redirect_destination"]:
|
if redirect["visit_id"] != result["redirect_destination"]:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -159,17 +169,22 @@ class SafariHistory(IOSExtraction):
|
|||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
if self.is_backup:
|
if self.is_backup:
|
||||||
for history_file in self._get_backup_files_from_manifest(
|
for relative_path in SAFARI_HISTORY_BACKUP_RELPATHS:
|
||||||
relative_path=SAFARI_HISTORY_BACKUP_RELPATH
|
for history_file in self._get_backup_files_from_manifest(
|
||||||
):
|
relative_path=relative_path
|
||||||
history_path = self._get_backup_file_from_id(history_file["file_id"])
|
):
|
||||||
|
history_path = self._get_backup_file_from_id(
|
||||||
|
history_file["file_id"]
|
||||||
|
)
|
||||||
|
|
||||||
if not history_path:
|
if not history_path:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self.log.info("Found Safari history database at path: %s", history_path)
|
self.log.info(
|
||||||
|
"Found Safari history database at path: %s", history_path
|
||||||
|
)
|
||||||
|
|
||||||
self._process_history_db(history_path)
|
self._process_history_db(history_path)
|
||||||
elif self.is_fs_dump:
|
elif self.is_fs_dump:
|
||||||
for history_path in self._get_fs_files_from_patterns(
|
for history_path in self._get_fs_files_from_patterns(
|
||||||
SAFARI_HISTORY_ROOT_PATHS
|
SAFARI_HISTORY_ROOT_PATHS
|
||||||
|
|||||||
@@ -76,8 +76,10 @@ class Shortcuts(IOSExtraction):
|
|||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
for result in self.results:
|
url_batches = [result["action_urls"] for result in self.results]
|
||||||
ioc_match = self.indicators.check_urls(result["action_urls"])
|
for result, ioc_match in zip(
|
||||||
|
self.results, self.indicators.check_url_batches(url_batches)
|
||||||
|
):
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
|
|||||||
@@ -85,17 +85,27 @@ class SMS(IOSExtraction):
|
|||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
url_batches = []
|
||||||
for result in self.results:
|
for result in self.results:
|
||||||
message_links = result.get("links", [])
|
message_links = result.get("links", [])
|
||||||
# Making sure not link was ignored
|
# Making sure not link was ignored
|
||||||
if message_links == []:
|
if message_links == []:
|
||||||
message_links = check_for_links(result.get("text", ""))
|
message_links = check_for_links(result.get("text", ""))
|
||||||
ioc_match = self.indicators.check_urls(message_links)
|
url_batches.append(message_links)
|
||||||
|
|
||||||
|
for result, ioc_match in zip(
|
||||||
|
self.results, self.indicators.check_url_batches(url_batches)
|
||||||
|
):
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def collect_url_results(self) -> None:
|
||||||
|
for message in self.results:
|
||||||
|
for url in message.get("links", []):
|
||||||
|
self.add_url_result(url, message.get("isodate"), "sms")
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
self._find_ios_database(backup_ids=SMS_BACKUP_IDS, root_paths=SMS_ROOT_PATHS)
|
self._find_ios_database(backup_ids=SMS_BACKUP_IDS, root_paths=SMS_ROOT_PATHS)
|
||||||
self.log.info("Found SMS database at path: %s", self.file_path)
|
self.log.info("Found SMS database at path: %s", self.file_path)
|
||||||
|
|||||||
@@ -70,6 +70,8 @@ class WebkitSessionResourceLog(IOSExtraction):
|
|||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
records = []
|
||||||
|
url_batches = []
|
||||||
for _, entries in self.results.items():
|
for _, entries in self.results.items():
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
source_domains = self._extract_domains(entry["redirect_source"])
|
source_domains = self._extract_domains(entry["redirect_source"])
|
||||||
@@ -94,36 +96,42 @@ class WebkitSessionResourceLog(IOSExtraction):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
ioc_match = self.indicators.check_urls(all_origins)
|
records.append((entry, source_domains, destination_domains))
|
||||||
if ioc_match:
|
url_batches.append(all_origins)
|
||||||
self.alertstore.critical(
|
|
||||||
ioc_match.message, "", entry, matched_indicator=ioc_match.ioc
|
|
||||||
)
|
|
||||||
|
|
||||||
redirect_path = ""
|
for record, ioc_match in zip(
|
||||||
if len(source_domains) > 0:
|
records, self.indicators.check_url_batches(url_batches)
|
||||||
redirect_path += "SOURCE: "
|
):
|
||||||
for idx, item in enumerate(source_domains):
|
if ioc_match:
|
||||||
source_domains[idx] = f'"{item}"'
|
entry, source_domains, destination_domains = record
|
||||||
|
self.alertstore.critical(
|
||||||
|
ioc_match.message, "", entry, matched_indicator=ioc_match.ioc
|
||||||
|
)
|
||||||
|
|
||||||
redirect_path += ", ".join(source_domains)
|
redirect_path = ""
|
||||||
redirect_path += " -> "
|
if len(source_domains) > 0:
|
||||||
|
redirect_path += "SOURCE: "
|
||||||
|
for idx, item in enumerate(source_domains):
|
||||||
|
source_domains[idx] = f'"{item}"'
|
||||||
|
|
||||||
redirect_path += f'ORIGIN: "{entry["origin"]}"'
|
redirect_path += ", ".join(source_domains)
|
||||||
|
redirect_path += " -> "
|
||||||
|
|
||||||
if len(destination_domains) > 0:
|
redirect_path += f'ORIGIN: "{entry["origin"]}"'
|
||||||
redirect_path += " -> "
|
|
||||||
redirect_path += "DESTINATION: "
|
|
||||||
for idx, item in enumerate(destination_domains):
|
|
||||||
destination_domains[idx] = f'"{item}"'
|
|
||||||
|
|
||||||
redirect_path += ", ".join(destination_domains)
|
if len(destination_domains) > 0:
|
||||||
|
redirect_path += " -> "
|
||||||
|
redirect_path += "DESTINATION: "
|
||||||
|
for idx, item in enumerate(destination_domains):
|
||||||
|
destination_domains[idx] = f'"{item}"'
|
||||||
|
|
||||||
self.alertstore.high(
|
redirect_path += ", ".join(destination_domains)
|
||||||
f"Found HTTP redirect between suspicious domains: {redirect_path}",
|
|
||||||
"",
|
self.alertstore.high(
|
||||||
entry,
|
f"Found HTTP redirect between suspicious domains: {redirect_path}",
|
||||||
)
|
"",
|
||||||
|
entry,
|
||||||
|
)
|
||||||
|
|
||||||
def _extract_browsing_stats(self, log_path):
|
def _extract_browsing_stats(self, log_path):
|
||||||
items = []
|
items = []
|
||||||
|
|||||||
@@ -4,7 +4,9 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import Optional
|
import os
|
||||||
|
import sqlite3
|
||||||
|
from typing import Dict, Optional
|
||||||
|
|
||||||
from mvt.common.module_types import (
|
from mvt.common.module_types import (
|
||||||
ModuleAtomicResult,
|
ModuleAtomicResult,
|
||||||
@@ -22,9 +24,71 @@ WHATSAPP_ROOT_PATHS = [
|
|||||||
"private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite",
|
"private/var/mobile/Containers/Shared/AppGroup/*/ChatStorage.sqlite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
WHATSAPP_LID_BACKUP_IDS = [
|
||||||
|
# SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-LID.sqlite"
|
||||||
|
"e794f6ffcc3c222535f47684a63d5178da3c4500",
|
||||||
|
]
|
||||||
|
WHATSAPP_LID_ROOT_PATHS = [
|
||||||
|
"private/var/mobile/Containers/Shared/AppGroup/*/LID.sqlite",
|
||||||
|
]
|
||||||
|
|
||||||
|
# WhatsApp records the mapping between a contact's LID and phone number
|
||||||
|
# identifiers in the ZWAPHONENUMBERLIDPAIR table. Depending on the WhatsApp
|
||||||
|
# version this lives in a dedicated LID.sqlite database or in
|
||||||
|
# ChatStorage.sqlite itself.
|
||||||
|
LID_PAIRS_QUERY = """
|
||||||
|
SELECT
|
||||||
|
ZLID AS "lid",
|
||||||
|
ZPHONENUMBER AS "phone_number",
|
||||||
|
ZTIMESTAMP AS "pair_timestamp"
|
||||||
|
FROM ZWAPHONENUMBERLIDPAIR;
|
||||||
|
"""
|
||||||
|
|
||||||
|
CHAT_SESSIONS_QUERY = """
|
||||||
|
SELECT
|
||||||
|
ZWACHATSESSION.Z_PK AS "session_pk",
|
||||||
|
ZWACHATSESSION.ZCONTACTJID AS "contact_jid",
|
||||||
|
ZWACHATSESSION.ZPARTNERNAME AS "partner_name",
|
||||||
|
ZWACHATSESSION.ZSESSIONTYPE AS "session_type",
|
||||||
|
ZWACHATSESSION.ZARCHIVED AS "archived",
|
||||||
|
ZWACHATSESSION.ZREMOVED AS "removed",
|
||||||
|
ZWACHATSESSION.ZMESSAGECOUNTER AS "message_counter",
|
||||||
|
ZWACHATSESSION.ZLASTMESSAGEDATE AS "last_message_date",
|
||||||
|
ZWAGROUPINFO.ZCREATIONDATE AS "group_creation_date",
|
||||||
|
MIN(ZWAMESSAGE.ZMESSAGEDATE) AS "first_stored_message_date",
|
||||||
|
MAX(ZWAMESSAGE.ZMESSAGEDATE) AS "last_stored_message_date",
|
||||||
|
COUNT(ZWAMESSAGE.Z_PK) AS "stored_message_count"
|
||||||
|
FROM ZWACHATSESSION
|
||||||
|
LEFT JOIN ZWAGROUPINFO
|
||||||
|
ON ZWACHATSESSION.ZGROUPINFO = ZWAGROUPINFO.Z_PK
|
||||||
|
LEFT JOIN ZWAMESSAGE
|
||||||
|
ON ZWAMESSAGE.ZCHATSESSION = ZWACHATSESSION.Z_PK
|
||||||
|
GROUP BY ZWACHATSESSION.Z_PK;
|
||||||
|
"""
|
||||||
|
|
||||||
|
CHAT_SESSION_DATE_FIELDS = [
|
||||||
|
"last_message_date",
|
||||||
|
"group_creation_date",
|
||||||
|
"first_stored_message_date",
|
||||||
|
"last_stored_message_date",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _describe_chat(record: ModuleAtomicResult) -> str:
|
||||||
|
jid = record.get("contact_jid") or "unknown"
|
||||||
|
name = record.get("partner_name")
|
||||||
|
identifier = record.get("partner_resolved_phone_number") or jid
|
||||||
|
label = f"'{name}' ({identifier})" if name else identifier
|
||||||
|
is_group = jid.endswith("@g.us") or record.get("group_creation_date")
|
||||||
|
if is_group:
|
||||||
|
return f"WhatsApp group chat {label}"
|
||||||
|
return f"WhatsApp chat with {label}"
|
||||||
|
|
||||||
|
|
||||||
class Whatsapp(IOSExtraction):
|
class Whatsapp(IOSExtraction):
|
||||||
"""This module extracts all WhatsApp messages containing links."""
|
"""This module extracts all WhatsApp messages containing links, as well
|
||||||
|
as per-chat records with the first and last interaction dates of each
|
||||||
|
conversation."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -45,6 +109,21 @@ class Whatsapp(IOSExtraction):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||||
|
if record.get("record_type") == "chat_session":
|
||||||
|
return self._serialize_chat_session(record)
|
||||||
|
if record.get("record_type") == "lid_phone_number_pair":
|
||||||
|
if not record.get("pair_timestamp"):
|
||||||
|
return []
|
||||||
|
return {
|
||||||
|
"timestamp": record["pair_timestamp"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "lid_pair_recorded",
|
||||||
|
"data": (
|
||||||
|
f"WhatsApp associated LID {record.get('lid')} with "
|
||||||
|
f"phone number {record.get('phone_number')}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
text = record.get("ZTEXT", "").replace("\n", "\\n")
|
text = record.get("ZTEXT", "").replace("\n", "\\n")
|
||||||
links_text = ""
|
links_text = ""
|
||||||
if record.get("links"):
|
if record.get("links"):
|
||||||
@@ -57,17 +136,67 @@ class Whatsapp(IOSExtraction):
|
|||||||
"data": f"'{text}' from {record.get('ZFROMJID', 'Unknown')}{links_text}",
|
"data": f"'{text}' from {record.get('ZFROMJID', 'Unknown')}{links_text}",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _serialize_chat_session(
|
||||||
|
self, record: ModuleAtomicResult
|
||||||
|
) -> ModuleSerializedResult:
|
||||||
|
records = []
|
||||||
|
chat = _describe_chat(record)
|
||||||
|
|
||||||
|
if record.get("group_creation_date"):
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record["group_creation_date"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "group_created",
|
||||||
|
"data": f"{chat} was created",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if record.get("first_stored_message_date"):
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record["first_stored_message_date"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "chat_first_message",
|
||||||
|
"data": f"First stored message in {chat}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# The chat session's own last-message date is authoritative: it can
|
||||||
|
# postdate the newest stored message if that message was deleted.
|
||||||
|
last_message_date = record.get("last_message_date") or record.get(
|
||||||
|
"last_stored_message_date"
|
||||||
|
)
|
||||||
|
if last_message_date:
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": last_message_date,
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "chat_last_message",
|
||||||
|
"data": f"Last message in {chat}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return records
|
||||||
|
|
||||||
def check_indicators(self) -> None:
|
def check_indicators(self) -> None:
|
||||||
if not self.indicators:
|
if not self.indicators:
|
||||||
return
|
return
|
||||||
|
|
||||||
for result in self.results:
|
url_batches = [result.get("links", []) for result in self.results]
|
||||||
ioc_match = self.indicators.check_urls(result.get("links", []))
|
for result, ioc_match in zip(
|
||||||
|
self.results, self.indicators.check_url_batches(url_batches)
|
||||||
|
):
|
||||||
if ioc_match:
|
if ioc_match:
|
||||||
self.alertstore.critical(
|
self.alertstore.critical(
|
||||||
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
ioc_match.message, "", result, matched_indicator=ioc_match.ioc
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def collect_url_results(self) -> None:
|
||||||
|
for message in self.results:
|
||||||
|
for url in message.get("links", []):
|
||||||
|
self.add_url_result(url, message.get("isodate"), "whatsapp")
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
self._find_ios_database(
|
self._find_ios_database(
|
||||||
backup_ids=WHATSAPP_BACKUP_IDS, root_paths=WHATSAPP_ROOT_PATHS
|
backup_ids=WHATSAPP_BACKUP_IDS, root_paths=WHATSAPP_ROOT_PATHS
|
||||||
@@ -139,7 +268,126 @@ class Whatsapp(IOSExtraction):
|
|||||||
message["links"] = list(set(filtered_links))
|
message["links"] = list(set(filtered_links))
|
||||||
self.results.append(message)
|
self.results.append(message)
|
||||||
|
|
||||||
|
total_messages = len(self.results)
|
||||||
|
lid_map = self._extract_lid_pairs(cur)
|
||||||
|
total_sessions = self._extract_chat_sessions(cur, lid_map)
|
||||||
|
|
||||||
cur.close()
|
cur.close()
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
self.log.info("Extracted a total of %d WhatsApp messages", len(self.results))
|
self.log.info(
|
||||||
|
"Extracted a total of %d WhatsApp messages, %d chat sessions "
|
||||||
|
"and %d LID-phone number pairs",
|
||||||
|
total_messages,
|
||||||
|
total_sessions,
|
||||||
|
len(lid_map),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _find_lid_db_path(self) -> Optional[str]:
|
||||||
|
for backup_id in WHATSAPP_LID_BACKUP_IDS:
|
||||||
|
file_path = self._get_backup_file_from_id(backup_id)
|
||||||
|
if file_path and os.path.exists(file_path):
|
||||||
|
return file_path
|
||||||
|
for found_path in self._get_fs_files_from_patterns(
|
||||||
|
WHATSAPP_LID_ROOT_PATHS
|
||||||
|
):
|
||||||
|
return found_path
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _extract_lid_pairs(self, chat_cur: sqlite3.Cursor) -> Dict[str, str]:
|
||||||
|
"""Extract the LID to phone number mapping from the dedicated
|
||||||
|
LID.sqlite database, falling back to the same table in
|
||||||
|
ChatStorage.sqlite. Returns a map of LID digits to phone number
|
||||||
|
digits."""
|
||||||
|
rows = []
|
||||||
|
lid_db_path = self._find_lid_db_path()
|
||||||
|
if lid_db_path:
|
||||||
|
self.log.info(
|
||||||
|
"Found WhatsApp LID database at path: %s", lid_db_path
|
||||||
|
)
|
||||||
|
lid_conn = self._open_sqlite_db(lid_db_path)
|
||||||
|
try:
|
||||||
|
lid_cur = lid_conn.cursor()
|
||||||
|
lid_cur.execute(LID_PAIRS_QUERY)
|
||||||
|
rows = lid_cur.fetchall()
|
||||||
|
lid_cur.close()
|
||||||
|
except sqlite3.DatabaseError as exc:
|
||||||
|
self.log.warning(
|
||||||
|
"Unable to extract WhatsApp LID-phone number pairs: %s",
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
lid_conn.close()
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
chat_cur.execute(LID_PAIRS_QUERY)
|
||||||
|
rows = chat_cur.fetchall()
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
self.log.info(
|
||||||
|
"No WhatsApp LID database found in this backup or "
|
||||||
|
"filesystem dump: LID chat identifiers cannot be "
|
||||||
|
"resolved to phone numbers"
|
||||||
|
)
|
||||||
|
|
||||||
|
lid_map: Dict[str, str] = {}
|
||||||
|
for lid, phone_number, pair_timestamp in rows:
|
||||||
|
record = {
|
||||||
|
"record_type": "lid_phone_number_pair",
|
||||||
|
"lid": lid,
|
||||||
|
"phone_number": phone_number,
|
||||||
|
"pair_timestamp": (
|
||||||
|
convert_mactime_to_iso(pair_timestamp) or None
|
||||||
|
)
|
||||||
|
if pair_timestamp
|
||||||
|
else None,
|
||||||
|
}
|
||||||
|
self.results.append(record)
|
||||||
|
|
||||||
|
if lid and phone_number:
|
||||||
|
lid_digits = str(lid).split("@")[0]
|
||||||
|
phone_digits = str(phone_number).split("@")[0].lstrip("+")
|
||||||
|
lid_map[lid_digits] = phone_digits
|
||||||
|
|
||||||
|
return lid_map
|
||||||
|
|
||||||
|
def _extract_chat_sessions(
|
||||||
|
self, cur: sqlite3.Cursor, lid_map: Dict[str, str]
|
||||||
|
) -> int:
|
||||||
|
"""Extract one record per chat session with the first and last
|
||||||
|
interaction dates of each conversation."""
|
||||||
|
try:
|
||||||
|
cur.execute(CHAT_SESSIONS_QUERY)
|
||||||
|
except sqlite3.OperationalError as exc:
|
||||||
|
self.log.warning(
|
||||||
|
"Unable to extract WhatsApp chat sessions: %s", exc
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
names = [description[0] for description in cur.description]
|
||||||
|
|
||||||
|
total_sessions = 0
|
||||||
|
for row in cur.fetchall():
|
||||||
|
session = dict(zip(names, row))
|
||||||
|
session["record_type"] = "chat_session"
|
||||||
|
|
||||||
|
session["partner_resolved_phone_number"] = None
|
||||||
|
jid = session.get("contact_jid") or ""
|
||||||
|
if jid.endswith("@lid"):
|
||||||
|
phone_digits = lid_map.get(jid.split("@")[0])
|
||||||
|
if phone_digits:
|
||||||
|
session["partner_resolved_phone_number"] = (
|
||||||
|
f"+{phone_digits}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for field in CHAT_SESSION_DATE_FIELDS:
|
||||||
|
if session.get(field):
|
||||||
|
session[field] = (
|
||||||
|
convert_mactime_to_iso(session[field]) or None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
session[field] = None
|
||||||
|
|
||||||
|
self.results.append(session)
|
||||||
|
total_sessions += 1
|
||||||
|
|
||||||
|
return total_sessions
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2026 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sqlite3
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from mvt.common.module import DatabaseNotFoundError
|
||||||
|
from mvt.common.module_types import (
|
||||||
|
ModuleAtomicResult,
|
||||||
|
ModuleResults,
|
||||||
|
ModuleSerializedResult,
|
||||||
|
)
|
||||||
|
from mvt.common.utils import convert_mactime_to_iso
|
||||||
|
|
||||||
|
from ..base import IOSExtraction
|
||||||
|
|
||||||
|
WHATSAPP_CONTACTS_BACKUP_IDS = [
|
||||||
|
# SHA-1 of "AppDomainGroup-group.net.whatsapp.WhatsApp.shared-ContactsV2.sqlite"
|
||||||
|
"b8548dc30aa1030df0ce18ef08b882cf7ab5212f",
|
||||||
|
]
|
||||||
|
WHATSAPP_CONTACTS_ROOT_PATHS = [
|
||||||
|
"private/var/mobile/Containers/Shared/AppGroup/*/ContactsV2.sqlite",
|
||||||
|
]
|
||||||
|
|
||||||
|
# WhatsApp's standard disappearing-messages timer values, in seconds.
|
||||||
|
DISAPPEARING_DURATION_LABELS = {
|
||||||
|
86400: "24 hours",
|
||||||
|
604800: "7 days",
|
||||||
|
1209600: "14 days",
|
||||||
|
2592000: "30 days",
|
||||||
|
7776000: "90 days",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Output field -> candidate columns in ZWAADDRESSBOOKCONTACT, in order of
|
||||||
|
# preference. WhatsApp renames columns across versions, so the query is built
|
||||||
|
# from the columns actually present in the database.
|
||||||
|
COLUMN_CANDIDATES = {
|
||||||
|
"whatsapp_id": ["ZWHATSAPPID"],
|
||||||
|
"lid": ["ZLID"],
|
||||||
|
"phone_number": ["ZPHONENUMBER"],
|
||||||
|
"localized_phone_number": ["ZLOCALIZEDPHONENUMBER"],
|
||||||
|
"full_name": ["ZFULLNAME"],
|
||||||
|
"given_name": ["ZGIVENNAME"],
|
||||||
|
"last_name": ["ZLASTNAME"],
|
||||||
|
"user_name": ["ZUSERNAME"],
|
||||||
|
"business_name": ["ZBUSINESSNAME"],
|
||||||
|
"about_text": ["ZABOUTTEXT"],
|
||||||
|
"about_emoji": ["ZABOUTEMOJI"],
|
||||||
|
"notes": ["ZNOTES"],
|
||||||
|
"disappearing_mode_duration": ["ZDISAPPEARINGMODEDURATION"],
|
||||||
|
"disappearing_mode_timestamp": ["ZDISAPPEARINGMODETIMESTAMP"],
|
||||||
|
"about_timestamp": ["ZABOUTTIMESTAMP"],
|
||||||
|
"about_expiration_timestamp": ["ZABOUTEXPIRATIONTIMESTAMP"],
|
||||||
|
"last_updated": ["ZLASTUPDATED"],
|
||||||
|
"phone_status": ["ZPHONESTATUS", "ZPHONENUMBERSTATUS"],
|
||||||
|
"sync_policy": ["ZSYNCPOLICY"],
|
||||||
|
}
|
||||||
|
|
||||||
|
STRING_FIELDS = [
|
||||||
|
"whatsapp_id",
|
||||||
|
"lid",
|
||||||
|
"phone_number",
|
||||||
|
"localized_phone_number",
|
||||||
|
"full_name",
|
||||||
|
"given_name",
|
||||||
|
"last_name",
|
||||||
|
"user_name",
|
||||||
|
"business_name",
|
||||||
|
"about_text",
|
||||||
|
"about_emoji",
|
||||||
|
"notes",
|
||||||
|
]
|
||||||
|
|
||||||
|
DATE_FIELDS = [
|
||||||
|
"disappearing_mode_timestamp",
|
||||||
|
"about_timestamp",
|
||||||
|
"about_expiration_timestamp",
|
||||||
|
"last_updated",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_string(value) -> Optional[str]:
|
||||||
|
# CoreData stores string attributes as UTF-8 blobs in some WhatsApp
|
||||||
|
# versions, so values can arrive as either bytes or str.
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.decode("utf-8", "replace")
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _label_duration(duration) -> str:
|
||||||
|
if not duration:
|
||||||
|
return "off"
|
||||||
|
return DISAPPEARING_DURATION_LABELS.get(
|
||||||
|
int(duration), f"{int(duration)} seconds"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _describe_contact(record: ModuleAtomicResult) -> str:
|
||||||
|
contact = (
|
||||||
|
record.get("whatsapp_id")
|
||||||
|
or record.get("lid")
|
||||||
|
or record.get("phone_number")
|
||||||
|
or "unknown"
|
||||||
|
)
|
||||||
|
full_name = record.get("full_name")
|
||||||
|
if full_name:
|
||||||
|
contact = f"{contact} ({full_name})"
|
||||||
|
return contact
|
||||||
|
|
||||||
|
|
||||||
|
class WhatsappContacts(IOSExtraction):
|
||||||
|
"""This module extracts WhatsApp contact records and per-contact
|
||||||
|
disappearing-messages settings from ContactsV2.sqlite.
|
||||||
|
|
||||||
|
ChatStorage.sqlite does not record the disappearing-messages state of 1:1
|
||||||
|
chats: the authoritative timer is stored on each contact record in this
|
||||||
|
database, alongside the mapping between a contact's LID and phone number
|
||||||
|
identifiers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
file_path: Optional[str] = None,
|
||||||
|
target_path: Optional[str] = None,
|
||||||
|
results_path: Optional[str] = None,
|
||||||
|
module_options: Optional[dict] = None,
|
||||||
|
log: logging.Logger = logging.getLogger(__name__),
|
||||||
|
results: Optional[ModuleResults] = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
file_path=file_path,
|
||||||
|
target_path=target_path,
|
||||||
|
results_path=results_path,
|
||||||
|
module_options=module_options,
|
||||||
|
log=log,
|
||||||
|
results=results,
|
||||||
|
)
|
||||||
|
|
||||||
|
def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult:
|
||||||
|
records = []
|
||||||
|
contact = _describe_contact(record)
|
||||||
|
|
||||||
|
if record.get("disappearing_mode_timestamp"):
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record["disappearing_mode_timestamp"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "disappearing_mode_set",
|
||||||
|
"data": (
|
||||||
|
f"WhatsApp disappearing messages timer set to "
|
||||||
|
f"'{record.get('disappearing_mode_label')}' "
|
||||||
|
f"for {contact}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if record.get("about_timestamp"):
|
||||||
|
data = f"WhatsApp about text of {contact} changed"
|
||||||
|
about_text = record.get("about_text")
|
||||||
|
if about_text:
|
||||||
|
data += f' to "{about_text}"'
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record["about_timestamp"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "about_changed",
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if record.get("about_expiration_timestamp"):
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record["about_expiration_timestamp"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "about_expiration",
|
||||||
|
"data": (
|
||||||
|
f"WhatsApp about text of {contact} scheduled "
|
||||||
|
f"to expire"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if record.get("last_updated"):
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"timestamp": record["last_updated"],
|
||||||
|
"module": self.__class__.__name__,
|
||||||
|
"event": "contact_last_updated",
|
||||||
|
"data": f"WhatsApp contact record for {contact} updated",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return records
|
||||||
|
|
||||||
|
def run(self) -> None:
|
||||||
|
try:
|
||||||
|
self._find_ios_database(
|
||||||
|
backup_ids=WHATSAPP_CONTACTS_BACKUP_IDS,
|
||||||
|
root_paths=WHATSAPP_CONTACTS_ROOT_PATHS,
|
||||||
|
)
|
||||||
|
except DatabaseNotFoundError:
|
||||||
|
self.log.warning(
|
||||||
|
"Unable to find the WhatsApp ContactsV2.sqlite database in "
|
||||||
|
"this backup or filesystem dump. WhatsApp disappearing "
|
||||||
|
"messages settings and contact records cannot be extracted. "
|
||||||
|
"This database is often missing from incremental backups."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.log.info(
|
||||||
|
"Found WhatsApp contacts database at path: %s", self.file_path
|
||||||
|
)
|
||||||
|
|
||||||
|
assert self.file_path is not None
|
||||||
|
conn = self._open_sqlite_db(self.file_path)
|
||||||
|
cur = conn.cursor()
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
cur.execute("PRAGMA table_info(ZWAADDRESSBOOKCONTACT)")
|
||||||
|
available_columns = {row[1] for row in cur.fetchall()}
|
||||||
|
except sqlite3.DatabaseError as exc:
|
||||||
|
self.log.error(
|
||||||
|
"Unable to read the ZWAADDRESSBOOKCONTACT table schema: %s",
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not available_columns:
|
||||||
|
self.log.warning(
|
||||||
|
"The WhatsApp contacts database does not contain a "
|
||||||
|
"ZWAADDRESSBOOKCONTACT table"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
selected = {}
|
||||||
|
for field, candidates in COLUMN_CANDIDATES.items():
|
||||||
|
for candidate in candidates:
|
||||||
|
if candidate in available_columns:
|
||||||
|
selected[field] = candidate
|
||||||
|
break
|
||||||
|
|
||||||
|
# A record with no duration column is "unknown", not "off": the
|
||||||
|
# timer state cannot be determined from this database version.
|
||||||
|
has_duration = "disappearing_mode_duration" in selected
|
||||||
|
if not has_duration:
|
||||||
|
self.log.warning(
|
||||||
|
"The ZDISAPPEARINGMODEDURATION column is not present in "
|
||||||
|
"this WhatsApp contacts database: disappearing messages "
|
||||||
|
"state is unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
columns = ["Z_PK"] + list(selected.values())
|
||||||
|
cur.execute(
|
||||||
|
f"SELECT {', '.join(columns)} FROM ZWAADDRESSBOOKCONTACT;"
|
||||||
|
)
|
||||||
|
fields = ["row_pk"] + list(selected.keys())
|
||||||
|
|
||||||
|
for row in cur:
|
||||||
|
record = dict(zip(fields, row))
|
||||||
|
|
||||||
|
for field in STRING_FIELDS:
|
||||||
|
if field in record:
|
||||||
|
record[field] = _decode_string(record[field])
|
||||||
|
else:
|
||||||
|
record[field] = None
|
||||||
|
|
||||||
|
for field in DATE_FIELDS:
|
||||||
|
if record.get(field) is not None:
|
||||||
|
record[field] = (
|
||||||
|
convert_mactime_to_iso(record[field]) or None
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
record[field] = None
|
||||||
|
|
||||||
|
duration = record.get("disappearing_mode_duration")
|
||||||
|
if has_duration:
|
||||||
|
record["disappearing_mode_is_on"] = bool(duration)
|
||||||
|
record["disappearing_mode_label"] = _label_duration(
|
||||||
|
duration
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
record["disappearing_mode_duration"] = None
|
||||||
|
record["disappearing_mode_is_on"] = None
|
||||||
|
record["disappearing_mode_label"] = None
|
||||||
|
|
||||||
|
record.setdefault("phone_status", None)
|
||||||
|
record.setdefault("sync_policy", None)
|
||||||
|
|
||||||
|
self.results.append(record)
|
||||||
|
finally:
|
||||||
|
cur.close()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
total_ephemeral = sum(
|
||||||
|
1 for record in self.results if record["disappearing_mode_is_on"]
|
||||||
|
)
|
||||||
|
self.log.info(
|
||||||
|
"Extracted a total of %d WhatsApp contacts (%d with disappearing "
|
||||||
|
"messages enabled)",
|
||||||
|
len(self.results),
|
||||||
|
total_ephemeral,
|
||||||
|
)
|
||||||
@@ -42,7 +42,8 @@ class NetBase(IOSExtraction):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _extract_net_data(self):
|
def _extract_net_data(self):
|
||||||
conn = sqlite3.connect(self.file_path)
|
assert self.file_path is not None
|
||||||
|
conn = self._open_sqlite_db(self.file_path)
|
||||||
conn.row_factory = sqlite3.Row
|
conn.row_factory = sqlite3.Row
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
try:
|
try:
|
||||||
@@ -237,9 +238,7 @@ class NetBase(IOSExtraction):
|
|||||||
"been truncated in the database)"
|
"been truncated in the database)"
|
||||||
)
|
)
|
||||||
|
|
||||||
self.alertstore.medium(
|
self.alertstore.medium(msg, proc["live_isodate"], proc)
|
||||||
msg, proc["live_isodate"], proc
|
|
||||||
)
|
|
||||||
|
|
||||||
if not proc["live_proc_id"]:
|
if not proc["live_proc_id"]:
|
||||||
self.log.info(
|
self.log.info(
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
from .base import SysdiagnoseExtraction
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import fnmatch
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tarfile
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from mvt.common.module import MVTModule, ModuleResults
|
||||||
|
|
||||||
|
|
||||||
|
class SysdiagnoseExtraction(MVTModule):
|
||||||
|
"""Base class for custom modules that analyze an iOS sysdiagnose."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
file_path: Optional[str] = None,
|
||||||
|
target_path: Optional[str] = None,
|
||||||
|
results_path: Optional[str] = None,
|
||||||
|
module_options: Optional[dict] = None,
|
||||||
|
log: logging.Logger = logging.getLogger(__name__),
|
||||||
|
results: Optional[ModuleResults] = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
file_path=file_path,
|
||||||
|
target_path=target_path,
|
||||||
|
results_path=results_path,
|
||||||
|
module_options=module_options,
|
||||||
|
log=log,
|
||||||
|
results=results,
|
||||||
|
)
|
||||||
|
self.parent_path: Optional[str] = None
|
||||||
|
self.files: list[str] = []
|
||||||
|
self.tar: Optional[tarfile.TarFile] = None
|
||||||
|
self.tar_files: list[str] = []
|
||||||
|
self.ips_files: list[dict[str, object]] = []
|
||||||
|
|
||||||
|
def from_sysdiagnose_folder(
|
||||||
|
self, target_path: str, sysdiagnose_files: list[str]
|
||||||
|
) -> None:
|
||||||
|
self.parent_path = Path(target_path).absolute().parent.as_posix()
|
||||||
|
self.files = sysdiagnose_files
|
||||||
|
|
||||||
|
def from_sysdiagnose_tar(
|
||||||
|
self, sysdiagnose_archive: tarfile.TarFile, sysdiagnose_files: list[str]
|
||||||
|
) -> None:
|
||||||
|
self.tar = sysdiagnose_archive
|
||||||
|
self.tar_files = sysdiagnose_files
|
||||||
|
|
||||||
|
def _extract_timezone(self):
|
||||||
|
"""Determine the sysdiagnose timezone from its diagnostic log."""
|
||||||
|
file_paths = self._get_files_by_pattern("*/sysdiagnose.log")
|
||||||
|
if not file_paths:
|
||||||
|
self.log.info(
|
||||||
|
"Unable to determine the timezone in which the sysdiagnose was "
|
||||||
|
"generated. Assuming UTC for logs without a timezone."
|
||||||
|
)
|
||||||
|
return timezone.utc
|
||||||
|
|
||||||
|
content = self._get_file_content(file_paths[0]).decode(
|
||||||
|
"utf-8", errors="replace"
|
||||||
|
)
|
||||||
|
filenames = re.findall(r"sysdiagnose_\S+?\.tar\.gz", content)
|
||||||
|
if not filenames:
|
||||||
|
self.log.info(
|
||||||
|
"Unable to determine the timezone in which the sysdiagnose was "
|
||||||
|
"generated. Assuming UTC for logs without a timezone."
|
||||||
|
)
|
||||||
|
return timezone.utc
|
||||||
|
|
||||||
|
timestamp = "_".join(
|
||||||
|
filenames[0].removesuffix(".tar.gz").split("_")[1:3]
|
||||||
|
)
|
||||||
|
sysdiagnose_timezone = datetime.strptime(
|
||||||
|
timestamp, "%Y.%m.%d_%H-%M-%S%z"
|
||||||
|
).tzinfo
|
||||||
|
self.log.info(
|
||||||
|
"Based on the sysdiagnose filename, assuming timezone %s for logs "
|
||||||
|
"without a timezone.",
|
||||||
|
sysdiagnose_timezone,
|
||||||
|
)
|
||||||
|
return sysdiagnose_timezone
|
||||||
|
|
||||||
|
def _get_files_by_pattern(self, pattern: str) -> list[str]:
|
||||||
|
file_names = self.tar_files if self.tar else self.files
|
||||||
|
return fnmatch.filter(file_names, pattern)
|
||||||
|
|
||||||
|
def _get_file_content(self, file_path: str) -> bytes:
|
||||||
|
if self.tar:
|
||||||
|
handle = self.tar.extractfile(self.tar.getmember(file_path))
|
||||||
|
else:
|
||||||
|
if self.parent_path is None:
|
||||||
|
raise RuntimeError("Sysdiagnose folder has not been initialized")
|
||||||
|
handle = open(os.path.join(self.parent_path, file_path), "rb")
|
||||||
|
|
||||||
|
if handle is None:
|
||||||
|
return b""
|
||||||
|
with handle:
|
||||||
|
return handle.read()
|
||||||
@@ -4,6 +4,8 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact
|
from mvt.android.artifacts.dumpsys_adb import DumpsysADBArtifact
|
||||||
|
from mvt.android.modules.bugreport.dumpsys_adb_state import DumpsysADBState
|
||||||
|
from mvt.common.alerts import AlertLevel
|
||||||
|
|
||||||
from ..utils import get_artifact
|
from ..utils import get_artifact
|
||||||
|
|
||||||
@@ -114,3 +116,119 @@ class TestDumpsysADBArtifact:
|
|||||||
assert key_store_entry["user"] == "user@laptop"
|
assert key_store_entry["user"] == "user@laptop"
|
||||||
assert key_store_entry["fingerprint"] == expected_fingerprint
|
assert key_store_entry["fingerprint"] == expected_fingerprint
|
||||||
assert key_store_entry["last_connected"] == "1628501829898"
|
assert key_store_entry["last_connected"] == "1628501829898"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDumpsysADBStateAlerts:
|
||||||
|
def test_no_androidqf_context_preserves_existing_behavior(self):
|
||||||
|
module = DumpsysADBState(
|
||||||
|
results=[
|
||||||
|
{
|
||||||
|
"user_keys": [
|
||||||
|
{
|
||||||
|
"key": b"QUJDRA==",
|
||||||
|
"user": "host@example",
|
||||||
|
"fingerprint": "fingerprint",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
module.check_indicators()
|
||||||
|
|
||||||
|
assert module.alertstore.alerts == []
|
||||||
|
|
||||||
|
def test_androidqf_trusted_keys_create_expected_alerts(self):
|
||||||
|
module = DumpsysADBState(
|
||||||
|
module_options={
|
||||||
|
"androidqf_acquisition": {
|
||||||
|
"started": "2025-06-20T18:00:00Z",
|
||||||
|
"adb_host_public_key": "QUJDRA== acquisition@host",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
results=[
|
||||||
|
{
|
||||||
|
"user_keys": [
|
||||||
|
{
|
||||||
|
"key": b"QUJDRA==",
|
||||||
|
"user": "acquisition@host",
|
||||||
|
"fingerprint": "acquisition-fingerprint",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": b"RUZHSA==",
|
||||||
|
"user": "other@host",
|
||||||
|
"fingerprint": "other-fingerprint",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": b"not-base64",
|
||||||
|
"user": "invalid@host",
|
||||||
|
"fingerprint": "",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"keystore": [
|
||||||
|
{
|
||||||
|
"key": b"QUJDRA==",
|
||||||
|
"user": "acquisition@host",
|
||||||
|
"fingerprint": "acquisition-fingerprint",
|
||||||
|
"last_connected": "1750266000000",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
module.check_indicators()
|
||||||
|
|
||||||
|
assert [alert.level for alert in module.alertstore.alerts] == [
|
||||||
|
AlertLevel.INFORMATIONAL,
|
||||||
|
AlertLevel.LOW,
|
||||||
|
AlertLevel.LOW,
|
||||||
|
]
|
||||||
|
informational, different, invalid = module.alertstore.alerts
|
||||||
|
assert "at least one day before" in informational.message
|
||||||
|
assert informational.event_time == "2025-06-18 17:00:00.000000"
|
||||||
|
assert "different from the AndroidQF acquisition host" in different.message
|
||||||
|
assert "invalid trusted ADB host key" in invalid.message
|
||||||
|
|
||||||
|
def test_missing_androidqf_host_key_creates_low_alert(self):
|
||||||
|
trusted_key = {
|
||||||
|
"key": b"QUJDRA==",
|
||||||
|
"user": "host@example",
|
||||||
|
"fingerprint": "fingerprint",
|
||||||
|
}
|
||||||
|
module = DumpsysADBState(
|
||||||
|
module_options={"androidqf_acquisition": {}},
|
||||||
|
results=[{"user_keys": [trusted_key]}],
|
||||||
|
)
|
||||||
|
|
||||||
|
module.check_indicators()
|
||||||
|
|
||||||
|
assert len(module.alertstore.alerts) == 1
|
||||||
|
assert module.alertstore.alerts[0].level == AlertLevel.LOW
|
||||||
|
assert "does not include its host key" in module.alertstore.alerts[0].message
|
||||||
|
|
||||||
|
def test_recent_acquisition_host_key_does_not_create_alert(self):
|
||||||
|
module = DumpsysADBState(
|
||||||
|
module_options={
|
||||||
|
"androidqf_acquisition": {
|
||||||
|
"started": "2025-06-20T18:00:00Z",
|
||||||
|
"adb_host_public_key": "QUJDRA== acquisition@host",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
results=[
|
||||||
|
{
|
||||||
|
"keystore": [
|
||||||
|
{
|
||||||
|
"key": b"QUJDRA==",
|
||||||
|
"user": "acquisition@host",
|
||||||
|
"fingerprint": "fingerprint",
|
||||||
|
"last_connected": "1750438800000",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
module.check_indicators()
|
||||||
|
|
||||||
|
assert module.alertstore.alerts == []
|
||||||
@@ -42,3 +42,22 @@ class TestDumpsysBatteryHistoryArtifact:
|
|||||||
assert len(dba.alertstore.alerts) == 0
|
assert len(dba.alertstore.alerts) == 0
|
||||||
dba.check_indicators()
|
dba.check_indicators()
|
||||||
assert len(dba.alertstore.alerts) == 2
|
assert len(dba.alertstore.alerts) == 2
|
||||||
|
|
||||||
|
def test_parsing_absolute_timestamps(self):
|
||||||
|
dba = DumpsysBatteryHistoryArtifact()
|
||||||
|
dba.parse(
|
||||||
|
"""07-15 20:27:39.431 (2) 100 +job=u0a123:"com.example/.ExampleJob"
|
||||||
|
07-15 20:27:40.431 (2) 100 -job=u0a123:"com.example/.ExampleJob"
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(dba.results) == 2
|
||||||
|
assert dba.results[0] == {
|
||||||
|
"time_elapsed": "07-15 20:27:39.431",
|
||||||
|
"event": "start_job",
|
||||||
|
"uid": "u0a123",
|
||||||
|
"package_name": "com.example",
|
||||||
|
"service": "com.example/.ExampleJob",
|
||||||
|
}
|
||||||
|
assert dba.results[1]["event"] == "end_job"
|
||||||
|
assert dba.results[1]["uid"] == "u0a123"
|
||||||
@@ -40,3 +40,22 @@ class TestDumpsysDBinfoArtifact:
|
|||||||
assert len(dbi.alertstore.alerts) == 0
|
assert len(dbi.alertstore.alerts) == 0
|
||||||
dbi.check_indicators()
|
dbi.check_indicators()
|
||||||
assert len(dbi.alertstore.alerts) == 5
|
assert len(dbi.alertstore.alerts) == 5
|
||||||
|
|
||||||
|
def test_parsing_month_day_timestamp_without_pid(self):
|
||||||
|
dbi = DumpsysDBInfoArtifact()
|
||||||
|
dbi.parse(
|
||||||
|
"""
|
||||||
|
Connection pool for /data/user/0/com.example/databases/current.db:
|
||||||
|
Most recently executed operations:
|
||||||
|
0: [07-15 20:27:39.431] executeForCursorWindow took 1ms - succeeded, sql="SELECT 1"
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert dbi.results == [
|
||||||
|
{
|
||||||
|
"isodate": "07-15 20:27:39.431",
|
||||||
|
"action": "executeForCursorWindow",
|
||||||
|
"sql": "SELECT 1",
|
||||||
|
"path": "/data/user/0/com.example/databases/current.db",
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -25,6 +25,25 @@ class TestDumpsysPackagesArtifact:
|
|||||||
== "com.samsung.android.provider.filterprovider"
|
== "com.samsung.android.provider.filterprovider"
|
||||||
)
|
)
|
||||||
assert dpa.results[0]["version_name"] == "5.0.07"
|
assert dpa.results[0]["version_name"] == "5.0.07"
|
||||||
|
assert dpa.results[0]["first_install_time"] == "2008-12-31 16:00:00"
|
||||||
|
assert dpa.results[0]["system"] is True
|
||||||
|
|
||||||
|
def test_parsing_system_flag(self):
|
||||||
|
system_details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
|
" pkgFlags=[ SYSTEM HAS_CODE ALLOW_CLEAR_USER_DATA ]"
|
||||||
|
)
|
||||||
|
third_party_details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
|
" pkgFlags=[ HAS_CODE ALLOW_BACKUP ]"
|
||||||
|
)
|
||||||
|
missing_flag_details = (
|
||||||
|
DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
|
" versionName=1.0"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert system_details["system"] is True
|
||||||
|
assert third_party_details["system"] is False
|
||||||
|
assert missing_flag_details["system"] is False
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
dpa = DumpsysPackagesArtifact()
|
dpa = DumpsysPackagesArtifact()
|
||||||
@@ -40,3 +59,43 @@ class TestDumpsysPackagesArtifact:
|
|||||||
assert len(dpa.alertstore.alerts) == 0
|
assert len(dpa.alertstore.alerts) == 0
|
||||||
dpa.check_indicators()
|
dpa.check_indicators()
|
||||||
assert len(dpa.alertstore.alerts) == 1
|
assert len(dpa.alertstore.alerts) == 1
|
||||||
|
|
||||||
|
def test_per_user_fields_use_primary_user(self):
|
||||||
|
details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
|
""" User 0: installed=true
|
||||||
|
firstInstallTime=2024-01-10 09:19:39
|
||||||
|
runtime permissions:
|
||||||
|
android.permission.CAMERA: granted=true
|
||||||
|
User 95: installed=false
|
||||||
|
firstInstallTime=1970-01-01 01:00:00
|
||||||
|
runtime permissions:
|
||||||
|
android.permission.CAMERA: granted=false
|
||||||
|
android.permission.RECORD_AUDIO: granted=false
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert details["first_install_time"] == "2024-01-10 09:19:39"
|
||||||
|
runtime_permissions = [
|
||||||
|
permission
|
||||||
|
for permission in details["permissions"]
|
||||||
|
if permission["type"] == "runtime"
|
||||||
|
]
|
||||||
|
assert runtime_permissions == [
|
||||||
|
{
|
||||||
|
"name": "android.permission.CAMERA",
|
||||||
|
"granted": True,
|
||||||
|
"type": "runtime",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_per_user_fields_fall_back_when_user_zero_is_missing(self):
|
||||||
|
details = DumpsysPackagesArtifact.parse_dumpsys_package_for_details(
|
||||||
|
""" User 10: installed=true
|
||||||
|
firstInstallTime=2024-02-10 09:19:39
|
||||||
|
runtime permissions:
|
||||||
|
android.permission.CAMERA: granted=true
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert details["first_install_time"] == "2024-02-10 09:19:39"
|
||||||
|
assert details["permissions"][-1]["name"] == "android.permission.CAMERA"
|
||||||
@@ -31,6 +31,44 @@ class TestDumpsysReceiversArtifact:
|
|||||||
== "com.android.storagemanager"
|
== "com.android.storagemanager"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_parsing_misindented_action(self):
|
||||||
|
dr = DumpsysReceiversArtifact()
|
||||||
|
data = """\
|
||||||
|
Receiver Resolver Table:
|
||||||
|
Non-Data Actions:
|
||||||
|
android.app.action.ENTER_CAR_MODE:
|
||||||
|
b5b40f6 com.google.android.projection.gearhead/.CarModeBroadcastReceiver
|
||||||
|
android.intent.action.MY_PACKAGE_REPLACED:
|
||||||
|
e7706c1 com.psycatgames.nhiegame/.ScheduledNotificationBootReceiver
|
||||||
|
"""
|
||||||
|
|
||||||
|
dr.parse(data)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][
|
||||||
|
"package_name"
|
||||||
|
]
|
||||||
|
== "com.psycatgames.nhiegame"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_parsing_misindented_first_action(self):
|
||||||
|
dr = DumpsysReceiversArtifact()
|
||||||
|
data = """\
|
||||||
|
Receiver Resolver Table:
|
||||||
|
Non-Data Actions:
|
||||||
|
android.intent.action.MY_PACKAGE_REPLACED:
|
||||||
|
e7706c1 com.psycatgames.nhiegame/.ScheduledNotificationBootReceiver
|
||||||
|
"""
|
||||||
|
|
||||||
|
dr.parse(data)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
dr.results["android.intent.action.MY_PACKAGE_REPLACED"][0][
|
||||||
|
"package_name"
|
||||||
|
]
|
||||||
|
== "com.psycatgames.nhiegame"
|
||||||
|
)
|
||||||
|
|
||||||
def test_ioc_check(self, indicator_file):
|
def test_ioc_check(self, indicator_file):
|
||||||
dr = DumpsysReceiversArtifact()
|
dr = DumpsysReceiversArtifact()
|
||||||
file = get_artifact("android_data/dumpsys_packages.txt")
|
file = get_artifact("android_data/dumpsys_packages.txt")
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import datetime
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
|
from mvt.android.artifacts.tombstone_crashes import TombstoneCrashArtifact
|
||||||
|
from mvt.android.parsers.proto.tombstone import Tombstone
|
||||||
|
|
||||||
from ..utils import get_artifact
|
from ..utils import get_artifact
|
||||||
|
|
||||||
@@ -42,6 +43,73 @@ class TestTombstoneCrashArtifact:
|
|||||||
assert len(tombstone_artifact.results) == 1
|
assert len(tombstone_artifact.results) == 1
|
||||||
self.validate_tombstone_result(tombstone_artifact.results[0])
|
self.validate_tombstone_result(tombstone_artifact.results[0])
|
||||||
|
|
||||||
|
def test_text_tombstone_preserves_abort_message(self):
|
||||||
|
tombstone_artifact = TombstoneCrashArtifact()
|
||||||
|
artifact_path = "android_data/bugreport/FS/data/tombstones/tombstone_00"
|
||||||
|
file = get_artifact(artifact_path)
|
||||||
|
with open(file, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
tombstone_artifact.parse(
|
||||||
|
os.path.basename(artifact_path),
|
||||||
|
datetime.datetime(2021, 9, 29, 17, 43, 49),
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert tombstone_artifact.results[0]["abort_message"] == (
|
||||||
|
"Check failed: payload.size() <= bytes_left "
|
||||||
|
"(payload.size()=99, bytes_left=51) "
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_protobuf_tombstone_preserves_abort_message_and_causes(self):
|
||||||
|
tombstone_artifact = TombstoneCrashArtifact()
|
||||||
|
artifact_path = "android_data/tombstone_process.pb"
|
||||||
|
file = get_artifact(artifact_path)
|
||||||
|
with open(file, "rb") as f:
|
||||||
|
tombstone = Tombstone().parse(f.read())
|
||||||
|
|
||||||
|
tombstone.abort_message = "synthetic abort reason"
|
||||||
|
tombstone_artifact.parse_protobuf(
|
||||||
|
os.path.basename(artifact_path),
|
||||||
|
datetime.datetime(2023, 4, 12, 12, 32, 40, 518290),
|
||||||
|
bytes(tombstone),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = tombstone_artifact.results[0]
|
||||||
|
assert result["abort_message"] == "synthetic abort reason"
|
||||||
|
assert result["causes"] == [
|
||||||
|
{
|
||||||
|
"human_readable": "null pointer dereference",
|
||||||
|
"memory_error": None,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_text_tombstone_keeps_crashing_thread(self):
|
||||||
|
tombstone_artifact = TombstoneCrashArtifact()
|
||||||
|
artifact_path = "android_data/tombstone_process.txt"
|
||||||
|
file = get_artifact(artifact_path)
|
||||||
|
with open(file, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
data += (
|
||||||
|
b"\npid: 25541, tid: 31896, name: worker-thread"
|
||||||
|
b" >>> /vendor/bin/other <<<\n"
|
||||||
|
)
|
||||||
|
tombstone_artifact.parse(
|
||||||
|
os.path.basename(artifact_path),
|
||||||
|
datetime.datetime(2023, 4, 12, 12, 32, 40, 518290),
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = tombstone_artifact.results[0]
|
||||||
|
assert result["pid"] == 25541
|
||||||
|
assert result["tid"] == 21307
|
||||||
|
assert result["process_name"] == "mtk.ape.decoder"
|
||||||
|
assert (
|
||||||
|
result["binary_path"]
|
||||||
|
== "/vendor/bin/hw/android.hardware.media.c2@1.2-mediatek"
|
||||||
|
)
|
||||||
|
|
||||||
@pytest.mark.skip(reason="Not implemented yet")
|
@pytest.mark.skip(reason="Not implemented yet")
|
||||||
def test_tombtone_kernel_parsing(self):
|
def test_tombtone_kernel_parsing(self):
|
||||||
tombstone_artifact = TombstoneCrashArtifact()
|
tombstone_artifact = TombstoneCrashArtifact()
|
||||||
|
|||||||
@@ -5,7 +5,10 @@
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from mvt.android.parsers.backup import (
|
from mvt.android.parsers.backup import (
|
||||||
|
AndroidBackupParsingError,
|
||||||
parse_ab_header,
|
parse_ab_header,
|
||||||
parse_backup_file,
|
parse_backup_file,
|
||||||
parse_tar_for_sms,
|
parse_tar_for_sms,
|
||||||
@@ -23,6 +26,14 @@ class TestBackupParsing:
|
|||||||
"encryption": None,
|
"encryption": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def test_parse_truncated_encrypted_header(self):
|
||||||
|
with pytest.raises(
|
||||||
|
AndroidBackupParsingError, match="Invalid encrypted backup header"
|
||||||
|
):
|
||||||
|
parse_backup_file(
|
||||||
|
b"ANDROID BACKUP\n5\n0\nAES-256\ntruncated", password="password"
|
||||||
|
)
|
||||||
|
|
||||||
def test_parsing_noencryption(self):
|
def test_parsing_noencryption(self):
|
||||||
file = get_artifact("android_backup/backup.ab")
|
file = get_artifact("android_backup/backup.ab")
|
||||||
with open(file, "rb") as f:
|
with open(file, "rb") as f:
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ class TestBugreportAnalysis:
|
|||||||
== "com.samsung.android.provider.filterprovider"
|
== "com.samsung.android.provider.filterprovider"
|
||||||
)
|
)
|
||||||
assert m.results[1]["package_name"] == "com.instagram.android"
|
assert m.results[1]["package_name"] == "com.instagram.android"
|
||||||
|
assert m.results[0]["installer"] == ""
|
||||||
|
assert m.results[1]["installer"] == "com.android.vending"
|
||||||
assert len(m.results[0]["permissions"]) == 4
|
assert len(m.results[0]["permissions"]) == 4
|
||||||
assert len(m.results[1]["permissions"]) == 32
|
assert len(m.results[1]["permissions"]) == 32
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,371 @@
|
|||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import click
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
from mvt.common.cli_plugins import (
|
||||||
|
ANDROID_CLI_PLUGIN_GROUP,
|
||||||
|
IOS_CLI_PLUGIN_GROUP,
|
||||||
|
BrokenPluginCommand,
|
||||||
|
load_cli_commands_option,
|
||||||
|
register_cli_commands_from_path,
|
||||||
|
register_cli_plugins,
|
||||||
|
register_installed_cli_commands,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
COMMAND_TEMPLATE = """
|
||||||
|
import click
|
||||||
|
|
||||||
|
|
||||||
|
@click.command({name!r})
|
||||||
|
@click.pass_context
|
||||||
|
def cli(ctx):
|
||||||
|
click.echo({message!r})
|
||||||
|
if ctx.obj:
|
||||||
|
click.echo(ctx.obj.get("marker", ""))
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _write_command(path, name, message="command ran"):
|
||||||
|
path.write_text(
|
||||||
|
COMMAND_TEMPLATE.format(name=name, message=message),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _make_group():
|
||||||
|
@click.group()
|
||||||
|
@load_cli_commands_option
|
||||||
|
@click.pass_context
|
||||||
|
def group(ctx):
|
||||||
|
ctx.ensure_object(dict)
|
||||||
|
ctx.obj["marker"] = "parent context"
|
||||||
|
|
||||||
|
return group
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_point(name, value, command=None, exception=None, distribution="plugin"):
|
||||||
|
def load():
|
||||||
|
if exception is not None:
|
||||||
|
raise exception
|
||||||
|
return command
|
||||||
|
|
||||||
|
dist = SimpleNamespace(metadata={"Name": distribution}, version="1.0")
|
||||||
|
return SimpleNamespace(name=name, value=value, load=load, dist=dist)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_command_option_registers_command_before_resolution(tmp_path):
|
||||||
|
command_path = _write_command(tmp_path / "hello.py", "hello")
|
||||||
|
group = _make_group()
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
group,
|
||||||
|
["--load-command", str(command_path), "hello"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "command ran" in result.output
|
||||||
|
assert "parent context" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_command_option_supports_folders_and_repeated_paths(tmp_path):
|
||||||
|
folder = tmp_path / "commands"
|
||||||
|
folder.mkdir()
|
||||||
|
_write_command(folder / "b.py", "second")
|
||||||
|
_write_command(folder / "a.py", "first")
|
||||||
|
_write_command(folder / ".hidden.py", "hidden")
|
||||||
|
_write_command(folder / "__init__.py", "init")
|
||||||
|
other = _write_command(tmp_path / "third.py", "third")
|
||||||
|
group = _make_group()
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
group,
|
||||||
|
[
|
||||||
|
"--load-command",
|
||||||
|
str(folder),
|
||||||
|
"--load-command",
|
||||||
|
str(other),
|
||||||
|
"--load-command",
|
||||||
|
str(other),
|
||||||
|
"--help",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "first" in result.output
|
||||||
|
assert "second" in result.output
|
||||||
|
assert "third" in result.output
|
||||||
|
assert "hidden" not in result.output
|
||||||
|
assert "init" not in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_loaded_command_participates_in_shell_completion(tmp_path):
|
||||||
|
command_path = _write_command(tmp_path / "hello.py", "hello")
|
||||||
|
group = _make_group()
|
||||||
|
words = f"group --load-command {command_path} he"
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
group,
|
||||||
|
[],
|
||||||
|
env={
|
||||||
|
"_GROUP_COMPLETE": "bash_complete",
|
||||||
|
"COMP_WORDS": words,
|
||||||
|
"COMP_CWORD": "3",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "plain,hello" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_command_import_and_contract_failures_are_usage_errors(tmp_path):
|
||||||
|
broken_path = tmp_path / "broken.py"
|
||||||
|
broken_path.write_text("raise RuntimeError('broken import')", encoding="utf-8")
|
||||||
|
missing_cli_path = tmp_path / "missing_cli.py"
|
||||||
|
missing_cli_path.write_text("value = 1", encoding="utf-8")
|
||||||
|
|
||||||
|
broken_result = CliRunner().invoke(
|
||||||
|
_make_group(),
|
||||||
|
["--load-command", str(broken_path), "broken"],
|
||||||
|
)
|
||||||
|
missing_cli_result = CliRunner().invoke(
|
||||||
|
_make_group(),
|
||||||
|
["--load-command", str(missing_cli_path), "missing-cli"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert broken_result.exit_code == 2
|
||||||
|
assert "broken import" in broken_result.output
|
||||||
|
assert missing_cli_result.exit_code == 2
|
||||||
|
assert "must export a Click command or group named 'cli'" in (
|
||||||
|
missing_cli_result.output
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_environment_command_failure_gets_broken_placeholder(tmp_path):
|
||||||
|
command_path = tmp_path / "broken_command.py"
|
||||||
|
command_path.write_text("raise RuntimeError('broken import')", encoding="utf-8")
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
registered = register_cli_commands_from_path(group, command_path)
|
||||||
|
|
||||||
|
assert registered == ["broken-command"]
|
||||||
|
assert isinstance(group.commands["broken-command"], BrokenPluginCommand)
|
||||||
|
result = CliRunner().invoke(group, ["broken-command"])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "broken import" in result.output
|
||||||
|
assert str(command_path) in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_environment_command_system_exit_gets_broken_placeholder(tmp_path):
|
||||||
|
command_path = tmp_path / "exiting_command.py"
|
||||||
|
command_path.write_text("raise SystemExit(7)", encoding="utf-8")
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
registered = register_cli_commands_from_path(group, command_path)
|
||||||
|
|
||||||
|
assert registered == ["exiting-command"]
|
||||||
|
assert isinstance(group.commands["exiting-command"], BrokenPluginCommand)
|
||||||
|
result = CliRunner().invoke(group, ["exiting-command"])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "Unable to import custom command" in result.output
|
||||||
|
assert result.output.rstrip().endswith(": 7")
|
||||||
|
|
||||||
|
|
||||||
|
def test_installed_entry_point_name_is_the_command_name(monkeypatch):
|
||||||
|
@click.command("internal-name")
|
||||||
|
def command():
|
||||||
|
click.echo("installed command ran")
|
||||||
|
|
||||||
|
entry_point = _entry_point(
|
||||||
|
"external-name",
|
||||||
|
"example_plugin:cli",
|
||||||
|
command=command,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
lambda **kwargs: [entry_point],
|
||||||
|
)
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
registered = register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||||
|
|
||||||
|
assert registered == ["external-name"]
|
||||||
|
assert "internal-name" not in group.commands
|
||||||
|
result = CliRunner().invoke(group, ["external-name"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert result.output == "installed command ran\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_broken_installed_plugin_does_not_break_cli(monkeypatch):
|
||||||
|
broken = _entry_point(
|
||||||
|
"broken",
|
||||||
|
"broken_plugin:cli",
|
||||||
|
exception=RuntimeError("missing dependency"),
|
||||||
|
distribution="broken-plugin",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
lambda **kwargs: [broken],
|
||||||
|
)
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||||
|
|
||||||
|
help_result = CliRunner().invoke(group, ["--help"])
|
||||||
|
assert help_result.exit_code == 0
|
||||||
|
assert "Warning: external command could not be loaded." in help_result.output
|
||||||
|
|
||||||
|
result = CliRunner().invoke(group, ["broken"])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "broken-plugin 1.0 (broken_plugin:cli)" in result.output
|
||||||
|
assert "RuntimeError: missing dependency" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_installed_plugin_system_exit_does_not_break_cli(monkeypatch):
|
||||||
|
exiting = _entry_point(
|
||||||
|
"exiting",
|
||||||
|
"exiting_plugin:cli",
|
||||||
|
exception=SystemExit(7),
|
||||||
|
distribution="exiting-plugin",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
lambda **kwargs: [exiting],
|
||||||
|
)
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||||
|
|
||||||
|
help_result = CliRunner().invoke(group, ["--help"])
|
||||||
|
assert help_result.exit_code == 0
|
||||||
|
result = CliRunner().invoke(group, ["exiting"])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "SystemExit: 7" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_click_entry_point_gets_broken_placeholder(monkeypatch):
|
||||||
|
invalid = _entry_point("invalid", "plugin:value", command=object())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
lambda **kwargs: [invalid],
|
||||||
|
)
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||||
|
|
||||||
|
assert isinstance(group.commands["invalid"], BrokenPluginCommand)
|
||||||
|
result = CliRunner().invoke(group, ["invalid"])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "must resolve to a Click command or group" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_entry_point_discovery_failure_does_not_break_group(monkeypatch, caplog):
|
||||||
|
def fail_discovery(**kwargs):
|
||||||
|
raise RuntimeError("invalid package metadata")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
fail_discovery,
|
||||||
|
)
|
||||||
|
group = click.Group()
|
||||||
|
|
||||||
|
registered = register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||||
|
|
||||||
|
assert registered == []
|
||||||
|
assert not group.commands
|
||||||
|
assert "Unable to discover external commands" in caplog.text
|
||||||
|
assert "invalid package metadata" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_builtin_and_first_external_command_win_collisions(monkeypatch, caplog):
|
||||||
|
@click.command("version")
|
||||||
|
def core_version():
|
||||||
|
pass
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
def first():
|
||||||
|
pass
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
def second():
|
||||||
|
pass
|
||||||
|
|
||||||
|
entry_points = [
|
||||||
|
_entry_point("duplicate", "z_plugin:cli", command=second, distribution="z"),
|
||||||
|
_entry_point("version", "plugin:version", command=first),
|
||||||
|
_entry_point("duplicate", "a_plugin:cli", command=first, distribution="a"),
|
||||||
|
]
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
lambda **kwargs: entry_points,
|
||||||
|
)
|
||||||
|
group = click.Group(commands={"version": core_version})
|
||||||
|
|
||||||
|
registered = register_installed_cli_commands(group, IOS_CLI_PLUGIN_GROUP)
|
||||||
|
|
||||||
|
assert registered == ["duplicate"]
|
||||||
|
assert group.commands["version"] is core_version
|
||||||
|
assert group.commands["duplicate"] is first
|
||||||
|
assert "the command name is already registered" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_command_cannot_replace_existing_command(tmp_path):
|
||||||
|
command_path = _write_command(tmp_path / "version.py", "version")
|
||||||
|
group = _make_group()
|
||||||
|
|
||||||
|
@group.command("version")
|
||||||
|
def core_version():
|
||||||
|
pass
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
group,
|
||||||
|
["--load-command", str(command_path), "version"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 2
|
||||||
|
assert "the command name is already registered" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_platform_entry_point_groups_and_environment_paths_are_separate(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
ios_path = _write_command(tmp_path / "ios.py", "ios-file")
|
||||||
|
android_path = _write_command(tmp_path / "android.py", "android-file")
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
def ios_package():
|
||||||
|
pass
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
def android_package():
|
||||||
|
pass
|
||||||
|
|
||||||
|
def entry_points(*, group):
|
||||||
|
if group == IOS_CLI_PLUGIN_GROUP:
|
||||||
|
return [_entry_point("ios-package", "ios_plugin:cli", ios_package)]
|
||||||
|
return [_entry_point("android-package", "android_plugin:cli", android_package)]
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"mvt.common.cli_plugins.importlib.metadata.entry_points",
|
||||||
|
entry_points,
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("TEST_IOS_COMMANDS", str(ios_path))
|
||||||
|
monkeypatch.setenv("TEST_ANDROID_COMMANDS", str(android_path))
|
||||||
|
ios_group = click.Group()
|
||||||
|
android_group = click.Group()
|
||||||
|
|
||||||
|
register_cli_plugins(
|
||||||
|
ios_group,
|
||||||
|
entry_point_group=IOS_CLI_PLUGIN_GROUP,
|
||||||
|
environment_variable="TEST_IOS_COMMANDS",
|
||||||
|
)
|
||||||
|
register_cli_plugins(
|
||||||
|
android_group,
|
||||||
|
entry_point_group=ANDROID_CLI_PLUGIN_GROUP,
|
||||||
|
environment_variable="TEST_ANDROID_COMMANDS",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert set(ios_group.commands) == {"ios-file", "ios-package"}
|
||||||
|
assert set(android_group.commands) == {"android-file", "android-package"}
|
||||||
@@ -42,6 +42,15 @@ class IndependentModule(RecordingModule):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class URLRecordingModule(RecordingModule):
|
||||||
|
def collect_url_results(self):
|
||||||
|
self.add_url_result(
|
||||||
|
"https://example.org/message",
|
||||||
|
"2026-07-29 12:00:00.000000",
|
||||||
|
"test-chat",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class CustomIOSBackupModule(RecordingModule):
|
class CustomIOSBackupModule(RecordingModule):
|
||||||
supported_commands = (("ios", "check-backup"),)
|
supported_commands = (("ios", "check-backup"),)
|
||||||
|
|
||||||
@@ -50,6 +59,10 @@ class CustomIOSFSModule(RecordingModule):
|
|||||||
supported_commands = (("ios", "check-fs"),)
|
supported_commands = (("ios", "check-fs"),)
|
||||||
|
|
||||||
|
|
||||||
|
class UnscopedCustomModule(RecordingModule):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class CustomDependsOnBuiltin(RecordingModule):
|
class CustomDependsOnBuiltin(RecordingModule):
|
||||||
supported_commands = (("ios", "check-backup"),)
|
supported_commands = (("ios", "check-backup"),)
|
||||||
dependencies = (FirstModule,)
|
dependencies = (FirstModule,)
|
||||||
@@ -83,6 +96,21 @@ class TestCommand:
|
|||||||
alerts = json.loads((tmp_path / "alerts.json").read_text())
|
alerts = json.loads((tmp_path / "alerts.json").read_text())
|
||||||
assert alerts[0]["event"]["payload"] == "\\xa8\\xa9"
|
assert alerts[0]["event"]["payload"] == "\\xa8\\xa9"
|
||||||
|
|
||||||
|
def test_stores_collected_urls(self, tmp_path):
|
||||||
|
cmd = RecordingCommand(results_path=str(tmp_path))
|
||||||
|
cmd.modules = [URLRecordingModule]
|
||||||
|
|
||||||
|
cmd.run()
|
||||||
|
|
||||||
|
assert json.loads((tmp_path / "urls.json").read_text()) == [
|
||||||
|
{
|
||||||
|
"url": "https://example.org/message",
|
||||||
|
"expanded_url": None,
|
||||||
|
"timestamp": "2026-07-29 12:00:00.000000",
|
||||||
|
"source": "test-chat",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
def test_modules_run_in_stable_topological_order(self):
|
def test_modules_run_in_stable_topological_order(self):
|
||||||
cmd = RecordingCommand()
|
cmd = RecordingCommand()
|
||||||
cmd.modules = [ThirdModule, IndependentModule, SecondModule, FirstModule]
|
cmd.modules = [ThirdModule, IndependentModule, SecondModule, FirstModule]
|
||||||
@@ -151,7 +179,11 @@ class TestCommand:
|
|||||||
cmd.platform = "ios"
|
cmd.platform = "ios"
|
||||||
cmd.name = "check-backup"
|
cmd.name = "check-backup"
|
||||||
cmd.modules = [FirstModule]
|
cmd.modules = [FirstModule]
|
||||||
cmd.custom_modules = [CustomIOSBackupModule, CustomIOSFSModule]
|
cmd.custom_modules = [
|
||||||
|
CustomIOSBackupModule,
|
||||||
|
CustomIOSFSModule,
|
||||||
|
UnscopedCustomModule,
|
||||||
|
]
|
||||||
|
|
||||||
assert [module.__name__ for module in cmd._ordered_modules()] == [
|
assert [module.__name__ for module in cmd._ordered_modules()] == [
|
||||||
"FirstModule",
|
"FirstModule",
|
||||||
|
|||||||
@@ -5,7 +5,9 @@
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
from mvt.common.config import settings
|
from mvt.common.config import settings
|
||||||
from mvt.common.indicators import Indicators
|
from mvt.common.indicators import Indicators
|
||||||
@@ -88,6 +90,128 @@ class TestIndicators:
|
|||||||
assert ind.check_url("https://goo.gl/maps/example") is None
|
assert ind.check_url("https://goo.gl/maps/example") is None
|
||||||
head_request.assert_not_called()
|
head_request.assert_not_called()
|
||||||
|
|
||||||
|
def test_check_url_batches_preserves_order(self, indicator_file):
|
||||||
|
ind = Indicators(log=logging)
|
||||||
|
ind.load_indicators_files([indicator_file], load_default=False)
|
||||||
|
|
||||||
|
matches = ind.check_url_batches(
|
||||||
|
[
|
||||||
|
[
|
||||||
|
"https://github.com",
|
||||||
|
"http://example.com/thisisbad",
|
||||||
|
"https://www.example.org/foobar",
|
||||||
|
],
|
||||||
|
["https://github.com", "https://www.example.org/foobar"],
|
||||||
|
[],
|
||||||
|
None,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matches[0]
|
||||||
|
assert matches[0].ioc.value == "http://example.com/thisisbad"
|
||||||
|
assert matches[1]
|
||||||
|
assert matches[1].ioc.value == "example.org"
|
||||||
|
assert matches[2] is None
|
||||||
|
assert matches[3] is None
|
||||||
|
|
||||||
|
def test_check_url_batches_deduplicates_and_limits_workers(
|
||||||
|
self, indicator_file, mocker
|
||||||
|
):
|
||||||
|
ind = Indicators(log=logging)
|
||||||
|
ind.load_indicators_files([indicator_file], load_default=False)
|
||||||
|
mocker.patch("mvt.common.indicators.URL_CHECK_MAX_WORKERS", 2)
|
||||||
|
|
||||||
|
barrier = threading.Barrier(2)
|
||||||
|
lock = threading.Lock()
|
||||||
|
calls = []
|
||||||
|
active = 0
|
||||||
|
max_active = 0
|
||||||
|
|
||||||
|
def head_request(url, timeout):
|
||||||
|
nonlocal active, max_active
|
||||||
|
with lock:
|
||||||
|
calls.append(url)
|
||||||
|
active += 1
|
||||||
|
max_active = max(max_active, active)
|
||||||
|
call_number = len(calls)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if call_number <= 2:
|
||||||
|
barrier.wait(timeout=5)
|
||||||
|
return mocker.Mock(status_code=200, headers={})
|
||||||
|
finally:
|
||||||
|
with lock:
|
||||||
|
active -= 1
|
||||||
|
|
||||||
|
mocker.patch("mvt.common.url.requests.head", side_effect=head_request)
|
||||||
|
urls = [
|
||||||
|
"https://bit.ly/one",
|
||||||
|
"https://tinyurl.com/two",
|
||||||
|
"https://t.co/three",
|
||||||
|
]
|
||||||
|
|
||||||
|
assert ind.check_url_batches([urls, [urls[0]]]) == [None, None]
|
||||||
|
assert sorted(calls) == sorted(urls)
|
||||||
|
assert max_active == 2
|
||||||
|
|
||||||
|
def test_check_url_batches_respects_disabled_network(
|
||||||
|
self, indicator_file, mocker
|
||||||
|
):
|
||||||
|
ind = Indicators(log=logging)
|
||||||
|
ind.load_indicators_files([indicator_file], load_default=False)
|
||||||
|
mocker.patch("mvt.common.indicators.settings.NETWORK_ACCESS_ALLOWED", False)
|
||||||
|
head_request = mocker.patch("mvt.common.url.requests.head")
|
||||||
|
|
||||||
|
assert ind.check_url_batches([["https://bit.ly/example"]]) == [None]
|
||||||
|
head_request.assert_not_called()
|
||||||
|
|
||||||
|
def test_check_url_batches_handles_nested_redirects_and_request_failures(
|
||||||
|
self, indicator_file, mocker
|
||||||
|
):
|
||||||
|
ind = Indicators(log=logging)
|
||||||
|
ind.load_indicators_files([indicator_file], load_default=False)
|
||||||
|
|
||||||
|
def head_request(url, timeout):
|
||||||
|
if url == "https://bit.ly/failure":
|
||||||
|
raise requests.Timeout()
|
||||||
|
if url == "https://tinyurl.com/nested":
|
||||||
|
return mocker.Mock(
|
||||||
|
status_code=301,
|
||||||
|
headers={"Location": "https://t.co/nested"},
|
||||||
|
)
|
||||||
|
if url == "https://t.co/nested":
|
||||||
|
return mocker.Mock(
|
||||||
|
status_code=302,
|
||||||
|
headers={"Location": "https://www.example.org/landing"},
|
||||||
|
)
|
||||||
|
raise AssertionError(f"Unexpected URL: {url}")
|
||||||
|
|
||||||
|
head = mocker.patch(
|
||||||
|
"mvt.common.url.requests.head", side_effect=head_request
|
||||||
|
)
|
||||||
|
|
||||||
|
matches = ind.check_url_batches(
|
||||||
|
[["https://bit.ly/failure"], ["https://tinyurl.com/nested"]]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matches[0] is None
|
||||||
|
assert matches[1]
|
||||||
|
assert matches[1].ioc.value == "example.org"
|
||||||
|
assert (
|
||||||
|
ind.get_expanded_url("https://tinyurl.com/nested")
|
||||||
|
== "https://www.example.org/landing"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
ind.get_expanded_url("https://t.co/nested")
|
||||||
|
== "https://www.example.org/landing"
|
||||||
|
)
|
||||||
|
assert ind.get_expanded_url("https://bit.ly/failure") is None
|
||||||
|
assert {call.args[0] for call in head.call_args_list} == {
|
||||||
|
"https://bit.ly/failure",
|
||||||
|
"https://tinyurl.com/nested",
|
||||||
|
"https://t.co/nested",
|
||||||
|
}
|
||||||
|
|
||||||
def test_check_file_hash(self, indicator_file):
|
def test_check_file_hash(self, indicator_file):
|
||||||
ind = Indicators(log=logging)
|
ind = Indicators(log=logging)
|
||||||
ind.load_indicators_files([indicator_file], load_default=False)
|
ind.load_indicators_files([indicator_file], load_default=False)
|
||||||
|
|||||||
@@ -125,12 +125,13 @@ def test_load_custom_modules_loads_env_folder_first(tmp_path, monkeypatch):
|
|||||||
assert [module.__name__ for module in modules] == ["EnvModule", "CliModule"]
|
assert [module.__name__ for module in modules] == ["EnvModule", "CliModule"]
|
||||||
|
|
||||||
|
|
||||||
def test_module_supports_command_defaults_to_all_commands(tmp_path):
|
def test_module_supports_command_requires_explicit_declaration(tmp_path, caplog):
|
||||||
module_path = _write_module(tmp_path / "custom.py", "DefaultModule")
|
module_path = _write_module(tmp_path / "custom.py", "DefaultModule")
|
||||||
module = load_custom_modules_from_path(str(module_path))[0]
|
module = load_custom_modules_from_path(str(module_path))[0]
|
||||||
|
|
||||||
assert module_supports_command(module, "ios", "check-backup")
|
assert not module_supports_command(module, "ios", "check-backup")
|
||||||
assert module_supports_command(module, "android", "check-bugreport")
|
assert not module_supports_command(module, "android", "check-bugreport")
|
||||||
|
assert "DefaultModule has no supported_commands" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_module_supports_command_honors_supported_commands(tmp_path):
|
def test_module_supports_command_honors_supported_commands(tmp_path):
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
from mvt.common.indicators import Indicators
|
from mvt.common.indicators import Indicators
|
||||||
from mvt.common.module import run_module
|
from mvt.common.module import run_module
|
||||||
@@ -21,6 +22,18 @@ class TestCalendarModule:
|
|||||||
assert len(m.alertstore.alerts) == 0
|
assert len(m.alertstore.alerts) == 0
|
||||||
assert m.results[0]["summary"] == "Super interesting meeting"
|
assert m.results[0]["summary"] == "Super interesting meeting"
|
||||||
|
|
||||||
|
def test_calendar_with_explicit_file_path(self):
|
||||||
|
backup_path = get_ios_backup_folder()
|
||||||
|
database_path = os.path.join(
|
||||||
|
backup_path, "20", "2041457d5fe04d39d0ab481178355df6781e6858"
|
||||||
|
)
|
||||||
|
m = Calendar(file_path=database_path)
|
||||||
|
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
assert len(m.results) == 1
|
||||||
|
assert m.results[0]["summary"] == "Super interesting meeting"
|
||||||
|
|
||||||
def test_calendar_detection(self, indicator_file):
|
def test_calendar_detection(self, indicator_file):
|
||||||
m = Calendar(target_path=get_ios_backup_folder())
|
m = Calendar(target_path=get_ios_backup_folder())
|
||||||
ind = Indicators(log=logging.getLogger())
|
ind = Indicators(log=logging.getLogger())
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2026 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
from mvt.common.module import run_module
|
||||||
|
from mvt.ios.modules.mixed.interactionc import InteractionC
|
||||||
|
from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts
|
||||||
|
|
||||||
|
from ..utils import get_ios_backup_folder
|
||||||
|
|
||||||
|
|
||||||
|
class TestInteractionCModule:
|
||||||
|
def test_extraction_with_whatsapp_contacts(self):
|
||||||
|
contacts = WhatsappContacts(target_path=get_ios_backup_folder())
|
||||||
|
run_module(contacts)
|
||||||
|
|
||||||
|
m = InteractionC(target_path=get_ios_backup_folder())
|
||||||
|
m.dependency_modules = {WhatsappContacts: contacts}
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
assert len(m.results) == 3
|
||||||
|
|
||||||
|
incoming = next(
|
||||||
|
r for r in m.results if r["sender_identifier"] == "100000000000001@lid"
|
||||||
|
)
|
||||||
|
assert incoming["direction"] == "INCOMING"
|
||||||
|
assert incoming["sender_resolved_phone_number"] == "+14155550100"
|
||||||
|
assert incoming["sender_resolved_name"] == "Alice Example"
|
||||||
|
|
||||||
|
outgoing = next(
|
||||||
|
r for r in m.results if r["direction"] == "OUTGOING"
|
||||||
|
)
|
||||||
|
assert outgoing["recipient_identifier"] == "+14155550100"
|
||||||
|
assert outgoing["recipient_resolved_name"] == "Alice Example"
|
||||||
|
assert outgoing["domain_resolved_phone_number"] == "+14155550100"
|
||||||
|
assert outgoing["domain_resolved_name"] == "Alice Example"
|
||||||
|
|
||||||
|
sms = next(
|
||||||
|
r for r in m.results if r["bundle_id"] == "com.apple.MobileSMS"
|
||||||
|
)
|
||||||
|
assert sms.get("sender_resolved_name") is None
|
||||||
|
assert sms["sender_display_name"] == "Bob Example"
|
||||||
|
|
||||||
|
events = [entry["data"] for entry in m.timeline]
|
||||||
|
assert (
|
||||||
|
"[net.whatsapp.WhatsApp] INCOMING from "
|
||||||
|
"Alice Example (+14155550100) to local user" in events
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
"[net.whatsapp.WhatsApp] OUTGOING from local user to "
|
||||||
|
"Alice Example (+14155550100)" in events
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
"[com.apple.MobileSMS] INCOMING from "
|
||||||
|
"Bob Example (+14155550101) to local user" in events
|
||||||
|
)
|
||||||
|
|
||||||
|
# The creation date is only serialized when it diverges from the
|
||||||
|
# start date; the SMS record was created 90 days after the event.
|
||||||
|
creation_events = [
|
||||||
|
entry
|
||||||
|
for entry in m.timeline
|
||||||
|
if entry["event"] == "interactions_creation_date"
|
||||||
|
]
|
||||||
|
assert len(creation_events) == 1
|
||||||
|
assert creation_events[0]["timestamp"] == "2025-12-09 12:26:40.000000"
|
||||||
|
assert creation_events[0]["data"] == (
|
||||||
|
"Interaction record created 90 days after the event: "
|
||||||
|
"[com.apple.MobileSMS] INCOMING from "
|
||||||
|
"Bob Example (+14155550101) to local user"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Per-contact aggregate dates use contact-centric data strings.
|
||||||
|
first_seen = [
|
||||||
|
entry
|
||||||
|
for entry in m.timeline
|
||||||
|
if entry["event"] == "first_incoming_sender_date"
|
||||||
|
]
|
||||||
|
assert len(first_seen) == 1
|
||||||
|
assert first_seen[0]["timestamp"] == "2025-09-03 13:46:40.000000"
|
||||||
|
assert first_seen[0]["data"] == (
|
||||||
|
"First incoming interaction from Bob Example (+14155550101)"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
"Last incoming interaction from Bob Example (+14155550101)"
|
||||||
|
in events
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_extraction_without_whatsapp_contacts(self):
|
||||||
|
# Without the WhatsappContacts dependency the module still runs, and
|
||||||
|
# unresolvable LIDs are shown as-is.
|
||||||
|
m = InteractionC(target_path=get_ios_backup_folder())
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
assert len(m.results) == 3
|
||||||
|
events = [entry["data"] for entry in m.timeline]
|
||||||
|
assert (
|
||||||
|
"[net.whatsapp.WhatsApp] INCOMING from "
|
||||||
|
"100000000000001@lid to local user" in events
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
"[net.whatsapp.WhatsApp] OUTGOING from local user to "
|
||||||
|
"+14155550100" in events
|
||||||
|
)
|
||||||
@@ -4,12 +4,44 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from mvt.common.indicators import Indicators
|
from mvt.common.indicators import Indicators
|
||||||
from mvt.common.module import run_module
|
from mvt.common.module import run_module
|
||||||
from mvt.ios.modules.mixed.safari_browserstate import SafariBrowserState
|
from mvt.ios.modules.mixed.safari_browserstate import SafariBrowserState
|
||||||
|
|
||||||
from ..utils import get_ios_backup_folder
|
from ..utils import add_backup_manifest_entry, get_ios_backup_folder
|
||||||
|
|
||||||
|
# fileID of HomeDomain::Library/Safari/BrowserState.db in the test backup.
|
||||||
|
DEFAULT_BROWSER_STATE_FILE_ID = "3a47b0981ed7c10f3e2800aa66bac96a3b5db28e"
|
||||||
|
PROFILE_UUID = "00000000-0000-4000-A000-000000000001"
|
||||||
|
PROFILE_BROWSER_STATE_FILE_ID = "bb00000000000000000000000000000000000001"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def backup_with_safari_profile(tmp_path):
|
||||||
|
"""An iTunes backup where a Safari profile has its own browser state."""
|
||||||
|
backup_path = tmp_path / "backup"
|
||||||
|
shutil.copytree(get_ios_backup_folder(), backup_path)
|
||||||
|
|
||||||
|
profile_db = (
|
||||||
|
backup_path / PROFILE_BROWSER_STATE_FILE_ID[:2] / PROFILE_BROWSER_STATE_FILE_ID
|
||||||
|
)
|
||||||
|
profile_db.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copyfile(
|
||||||
|
backup_path / DEFAULT_BROWSER_STATE_FILE_ID[:2] / DEFAULT_BROWSER_STATE_FILE_ID,
|
||||||
|
profile_db,
|
||||||
|
)
|
||||||
|
add_backup_manifest_entry(
|
||||||
|
backup_path,
|
||||||
|
PROFILE_BROWSER_STATE_FILE_ID,
|
||||||
|
"AppDomain-com.apple.mobilesafari",
|
||||||
|
f"Library/Safari/Profiles/{PROFILE_UUID}/BrowserState.db",
|
||||||
|
)
|
||||||
|
|
||||||
|
return str(backup_path)
|
||||||
|
|
||||||
|
|
||||||
class TestSafariBrowserStateModule:
|
class TestSafariBrowserStateModule:
|
||||||
@@ -21,6 +53,15 @@ class TestSafariBrowserStateModule:
|
|||||||
assert len(m.timeline) == 1
|
assert len(m.timeline) == 1
|
||||||
assert len(m.alertstore.alerts) == 0
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
|
def test_parsing_backup_with_profile(self, backup_with_safari_profile):
|
||||||
|
m = SafariBrowserState(target_path=backup_with_safari_profile)
|
||||||
|
m.is_backup = True
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
# Both the default profile and the named profile are extracted.
|
||||||
|
assert len(m.results) == 2
|
||||||
|
assert len({result["safari_browser_state_db"] for result in m.results}) == 2
|
||||||
|
|
||||||
def test_detection(self, indicator_file):
|
def test_detection(self, indicator_file):
|
||||||
m = SafariBrowserState(target_path=get_ios_backup_folder())
|
m = SafariBrowserState(target_path=get_ios_backup_folder())
|
||||||
m.is_backup = True
|
m.is_backup = True
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from mvt.common.indicators import Indicators
|
||||||
|
from mvt.common.module import run_module
|
||||||
|
from mvt.ios.modules.mixed.safari_history import SafariHistory
|
||||||
|
|
||||||
|
from ..utils import add_backup_manifest_entry, get_ios_backup_folder
|
||||||
|
|
||||||
|
# fileID of HomeDomain::Library/Safari/History.db in the test backup.
|
||||||
|
DEFAULT_HISTORY_FILE_ID = "1a0e7afc19d307da602ccdcece51af33afe92c53"
|
||||||
|
PROFILE_UUID = "00000000-0000-4000-A000-000000000001"
|
||||||
|
PROFILE_HISTORY_FILE_ID = "aa00000000000000000000000000000000000001"
|
||||||
|
|
||||||
|
# example.org is already a test indicator, so use domains that do not match.
|
||||||
|
DEFAULT_URL = "https://default.example.net/visited-page"
|
||||||
|
PROFILE_URL = "https://profile.example.net/visited-page"
|
||||||
|
|
||||||
|
|
||||||
|
def create_history_db(path, url):
|
||||||
|
"""Create a minimal Safari History.db holding a single visit."""
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
conn.executescript(
|
||||||
|
"""
|
||||||
|
CREATE TABLE history_items (id INTEGER PRIMARY KEY, url TEXT);
|
||||||
|
CREATE TABLE history_visits (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
history_item INTEGER,
|
||||||
|
visit_time REAL,
|
||||||
|
redirect_source INTEGER,
|
||||||
|
redirect_destination INTEGER
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO history_items VALUES (1, ?);", (url,))
|
||||||
|
conn.execute("INSERT INTO history_visits VALUES (1, 1, 726100000.0, NULL, NULL);")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def backup_with_safari_profile(tmp_path):
|
||||||
|
"""An iTunes backup where Safari has both a default and a named profile."""
|
||||||
|
backup_path = tmp_path / "backup"
|
||||||
|
shutil.copytree(get_ios_backup_folder(), backup_path)
|
||||||
|
|
||||||
|
# The default profile's database ships empty, so give it a visit to make
|
||||||
|
# sure the profile lookup does not replace the pre-existing one.
|
||||||
|
create_history_db(
|
||||||
|
backup_path / DEFAULT_HISTORY_FILE_ID[:2] / DEFAULT_HISTORY_FILE_ID,
|
||||||
|
DEFAULT_URL,
|
||||||
|
)
|
||||||
|
|
||||||
|
create_history_db(
|
||||||
|
backup_path / PROFILE_HISTORY_FILE_ID[:2] / PROFILE_HISTORY_FILE_ID,
|
||||||
|
PROFILE_URL,
|
||||||
|
)
|
||||||
|
add_backup_manifest_entry(
|
||||||
|
backup_path,
|
||||||
|
PROFILE_HISTORY_FILE_ID,
|
||||||
|
"AppDomain-com.apple.mobilesafari",
|
||||||
|
f"Library/Safari/Profiles/{PROFILE_UUID}/History.db",
|
||||||
|
)
|
||||||
|
|
||||||
|
return str(backup_path)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fs_dump_with_safari_profile(tmp_path):
|
||||||
|
"""A filesystem dump where Safari has both a default and a named profile."""
|
||||||
|
safari_path = tmp_path / "private" / "var" / "mobile" / "Library" / "Safari"
|
||||||
|
profile_path = safari_path / "Profiles" / PROFILE_UUID
|
||||||
|
profile_path.mkdir(parents=True)
|
||||||
|
|
||||||
|
create_history_db(safari_path / "History.db", DEFAULT_URL)
|
||||||
|
create_history_db(profile_path / "History.db", PROFILE_URL)
|
||||||
|
|
||||||
|
return str(tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSafariHistoryModule:
|
||||||
|
def test_parsing(self):
|
||||||
|
m = SafariHistory(target_path=get_ios_backup_folder())
|
||||||
|
m.is_backup = True
|
||||||
|
run_module(m)
|
||||||
|
assert len(m.results) == 0
|
||||||
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
|
def test_parsing_backup_with_profile(self, backup_with_safari_profile):
|
||||||
|
m = SafariHistory(target_path=backup_with_safari_profile)
|
||||||
|
m.is_backup = True
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
# Both the default profile and the named profile are extracted.
|
||||||
|
assert len(m.results) == 2
|
||||||
|
assert {result["url"] for result in m.results} == {DEFAULT_URL, PROFILE_URL}
|
||||||
|
assert len({result["safari_history_db"] for result in m.results}) == 2
|
||||||
|
|
||||||
|
def test_parsing_fs_dump_with_profile(self, fs_dump_with_safari_profile):
|
||||||
|
m = SafariHistory(target_path=fs_dump_with_safari_profile)
|
||||||
|
m.is_fs_dump = True
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
assert len(m.results) == 2
|
||||||
|
assert {result["url"] for result in m.results} == {DEFAULT_URL, PROFILE_URL}
|
||||||
|
|
||||||
|
def test_redirect_ids_are_scoped_to_database(self, fs_dump_with_safari_profile):
|
||||||
|
safari_path = (
|
||||||
|
Path(fs_dump_with_safari_profile)
|
||||||
|
/ "private"
|
||||||
|
/ "var"
|
||||||
|
/ "mobile"
|
||||||
|
/ "Library"
|
||||||
|
/ "Safari"
|
||||||
|
)
|
||||||
|
with sqlite3.connect(safari_path / "History.db") as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE history_items SET url = ? WHERE id = 1;",
|
||||||
|
("http://safe.example.com/start",),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO history_items VALUES (2, ?);",
|
||||||
|
("https://safe.example.com/end",),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE history_visits SET redirect_destination = 2 WHERE id = 1;"
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO history_visits VALUES (2, 2, 726100000.1, 1, NULL);"
|
||||||
|
)
|
||||||
|
|
||||||
|
profile_db = safari_path / "Profiles" / PROFILE_UUID / "History.db"
|
||||||
|
with sqlite3.connect(profile_db) as conn:
|
||||||
|
# Visit IDs are local to each database and commonly overlap.
|
||||||
|
conn.execute("UPDATE history_visits SET id = 2 WHERE id = 1;")
|
||||||
|
|
||||||
|
m = SafariHistory(target_path=fs_dump_with_safari_profile)
|
||||||
|
m.is_fs_dump = True
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
assert len(m.results) == 3
|
||||||
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
|
def test_detection_in_profile(self, backup_with_safari_profile, indicator_file):
|
||||||
|
"""An indicator only visited inside a Safari profile still alerts."""
|
||||||
|
m = SafariHistory(target_path=backup_with_safari_profile)
|
||||||
|
m.is_backup = True
|
||||||
|
ind = Indicators(log=logging.getLogger())
|
||||||
|
ind.parse_stix2(indicator_file)
|
||||||
|
ind.ioc_collections[0]["domains"].append("profile.example.net")
|
||||||
|
m.indicators = ind
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
assert len(m.alertstore.alerts) == 1
|
||||||
|
assert m.alertstore.alerts[0].event["url"] == PROFILE_URL
|
||||||
@@ -18,6 +18,14 @@ class TestSMSModule:
|
|||||||
run_module(m)
|
run_module(m)
|
||||||
assert len(m.results) == 1
|
assert len(m.results) == 1
|
||||||
assert len(m.timeline) == 2
|
assert len(m.timeline) == 2
|
||||||
|
assert m.url_results == [
|
||||||
|
{
|
||||||
|
"url": "https://badbadbad.example.org/",
|
||||||
|
"expanded_url": None,
|
||||||
|
"timestamp": "2019-08-29 23:13:30.000000",
|
||||||
|
"source": "sms",
|
||||||
|
}
|
||||||
|
]
|
||||||
assert len(m.alertstore.alerts) == 0
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
def test_detection(self, indicator_file):
|
def test_detection(self, indicator_file):
|
||||||
@@ -29,3 +37,31 @@ class TestSMSModule:
|
|||||||
m.indicators = ind
|
m.indicators = ind
|
||||||
run_module(m)
|
run_module(m)
|
||||||
assert len(m.alertstore.alerts) == 1
|
assert len(m.alertstore.alerts) == 1
|
||||||
|
|
||||||
|
def test_detection_batches_urls_and_preserves_event(self, indicator_file, mocker):
|
||||||
|
results = [
|
||||||
|
{
|
||||||
|
"text": "first",
|
||||||
|
"links": ["http://example.com/thisisbad"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"text": "second",
|
||||||
|
"links": ["https://github.com"],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
m = SMS(results=results)
|
||||||
|
ind = Indicators(log=logging.getLogger())
|
||||||
|
ind.parse_stix2(indicator_file)
|
||||||
|
batch_check = mocker.spy(ind, "check_url_batches")
|
||||||
|
m.indicators = ind
|
||||||
|
|
||||||
|
m.check_indicators()
|
||||||
|
|
||||||
|
batch_check.assert_called_once_with(
|
||||||
|
[
|
||||||
|
["http://example.com/thisisbad"],
|
||||||
|
["https://github.com"],
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert len(m.alertstore.alerts) == 1
|
||||||
|
assert m.alertstore.alerts[0].event is results[0]
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2023 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import plistlib
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
from mvt.ios.modules.base import IOSExtraction
|
||||||
|
from mvt.ios.modules.fs.analytics import Analytics
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path):
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def test_open_sqlite_reads_wal_without_modifying_evidence(tmp_path):
|
||||||
|
live_path = tmp_path / "live.db"
|
||||||
|
evidence_path = tmp_path / "evidence.db"
|
||||||
|
conn = sqlite3.connect(live_path)
|
||||||
|
conn.execute("PRAGMA journal_mode=WAL;")
|
||||||
|
conn.execute("PRAGMA wal_autocheckpoint=0;")
|
||||||
|
conn.execute("CREATE TABLE records (value TEXT);")
|
||||||
|
conn.commit()
|
||||||
|
conn.execute("INSERT INTO records VALUES ('from wal');")
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
shutil.copy2(live_path, evidence_path)
|
||||||
|
shutil.copy2(str(live_path) + "-wal", str(evidence_path) + "-wal")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
evidence_hash = _sha256(evidence_path)
|
||||||
|
wal_hash = _sha256(tmp_path / "evidence.db-wal")
|
||||||
|
module = IOSExtraction(file_path=str(evidence_path))
|
||||||
|
read_conn = module._open_sqlite_db(str(evidence_path))
|
||||||
|
rows = read_conn.execute("SELECT value FROM records;").fetchall()
|
||||||
|
read_conn.close()
|
||||||
|
|
||||||
|
assert rows == [("from wal",)]
|
||||||
|
assert _sha256(evidence_path) == evidence_hash
|
||||||
|
assert _sha256(tmp_path / "evidence.db-wal") == wal_hash
|
||||||
|
assert not os.path.exists(str(evidence_path) + "-shm")
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_preserves_source_database(tmp_path):
|
||||||
|
database_path = tmp_path / "source.db"
|
||||||
|
conn = sqlite3.connect(database_path)
|
||||||
|
conn.execute("CREATE TABLE records (value TEXT);")
|
||||||
|
conn.execute("INSERT INTO records VALUES ('preserved');")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
source_hash = _sha256(database_path)
|
||||||
|
|
||||||
|
module = IOSExtraction(file_path=str(database_path))
|
||||||
|
module._recover_sqlite_db_if_needed(str(database_path), forced=True)
|
||||||
|
recovered_conn = module._open_sqlite_db(str(database_path))
|
||||||
|
rows = recovered_conn.execute("SELECT value FROM records;").fetchall()
|
||||||
|
recovered_conn.close()
|
||||||
|
|
||||||
|
assert rows == [("preserved",)]
|
||||||
|
assert _sha256(database_path) == source_hash
|
||||||
|
assert not os.path.exists(str(database_path) + ".bak")
|
||||||
|
|
||||||
|
|
||||||
|
def test_analytics_skips_empty_rows_and_continues(tmp_path):
|
||||||
|
database_path = tmp_path / "analytics.db"
|
||||||
|
conn = sqlite3.connect(database_path)
|
||||||
|
for table in ("hard_failures", "soft_failures", "all_events"):
|
||||||
|
conn.execute(f"CREATE TABLE {table} (timestamp REAL, data BLOB);")
|
||||||
|
conn.execute("INSERT INTO hard_failures VALUES (NULL, NULL);")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO soft_failures VALUES (?, ?);",
|
||||||
|
(1.0, plistlib.dumps({"event": "valid"})),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
module = Analytics(file_path=str(database_path))
|
||||||
|
module._extract_analytics_data()
|
||||||
|
|
||||||
|
assert len(module.results) == 1
|
||||||
|
assert module.results[0]["event"] == "valid"
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2026 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from mvt.common.indicators import Indicators
|
||||||
|
from mvt.common.module import run_module
|
||||||
|
from mvt.ios.modules.mixed.whatsapp import Whatsapp
|
||||||
|
|
||||||
|
from ..utils import get_ios_backup_folder
|
||||||
|
|
||||||
|
|
||||||
|
def test_extraction():
|
||||||
|
m = Whatsapp(target_path=get_ios_backup_folder())
|
||||||
|
run_module(m)
|
||||||
|
|
||||||
|
messages = [r for r in m.results if "ZTEXT" in r]
|
||||||
|
sessions = [r for r in m.results if r.get("record_type") == "chat_session"]
|
||||||
|
pairs = [
|
||||||
|
r for r in m.results
|
||||||
|
if r.get("record_type") == "lid_phone_number_pair"
|
||||||
|
]
|
||||||
|
assert len(messages) == 3
|
||||||
|
assert len(sessions) == 2
|
||||||
|
assert len(pairs) == 1
|
||||||
|
|
||||||
|
assert pairs[0]["lid"] == "100000000000001"
|
||||||
|
assert pairs[0]["phone_number"] == "14155550100"
|
||||||
|
assert pairs[0]["pair_timestamp"] == "2025-08-25 07:33:20.000000"
|
||||||
|
|
||||||
|
linked = next(r for r in messages if r.get("links"))
|
||||||
|
assert linked["links"] == ["https://example.org/news"]
|
||||||
|
|
||||||
|
alice = next(s for s in sessions if s["partner_name"] == "Alice Example")
|
||||||
|
assert alice["contact_jid"] == "100000000000001@lid"
|
||||||
|
assert alice["partner_resolved_phone_number"] == "+14155550100"
|
||||||
|
assert alice["first_stored_message_date"] == "2025-08-27 15:06:40.000000"
|
||||||
|
assert alice["last_message_date"] == "2025-08-28 18:53:20.000000"
|
||||||
|
assert alice["group_creation_date"] is None
|
||||||
|
assert alice["stored_message_count"] == 2
|
||||||
|
|
||||||
|
group = next(s for s in sessions if s["partner_name"] == "Example Group")
|
||||||
|
assert group["group_creation_date"] == "2025-08-21 20:13:20.000000"
|
||||||
|
assert group["first_stored_message_date"] == "2025-08-29 22:40:00.000000"
|
||||||
|
# The last stored message predates the session's own last-message date:
|
||||||
|
# the newest message in this chat was deleted.
|
||||||
|
assert group["last_stored_message_date"] == "2025-08-29 22:40:00.000000"
|
||||||
|
assert group["last_message_date"] == "2025-08-31 02:26:40.000000"
|
||||||
|
|
||||||
|
# 3 message events, first/last per chat, the group creation and the
|
||||||
|
# LID-phone number pair.
|
||||||
|
assert len(m.timeline) == 9
|
||||||
|
events = {
|
||||||
|
(entry["event"], entry["timestamp"]): entry["data"]
|
||||||
|
for entry in m.timeline
|
||||||
|
}
|
||||||
|
# Alice's session is keyed by LID but labelled with the phone number
|
||||||
|
# resolved through LID.sqlite.
|
||||||
|
assert events[("chat_first_message", "2025-08-27 15:06:40.000000")] == (
|
||||||
|
"First stored message in WhatsApp chat with "
|
||||||
|
"'Alice Example' (+14155550100)"
|
||||||
|
)
|
||||||
|
assert events[("chat_last_message", "2025-08-28 18:53:20.000000")] == (
|
||||||
|
"Last message in WhatsApp chat with "
|
||||||
|
"'Alice Example' (+14155550100)"
|
||||||
|
)
|
||||||
|
assert events[("lid_pair_recorded", "2025-08-25 07:33:20.000000")] == (
|
||||||
|
"WhatsApp associated LID 100000000000001 with "
|
||||||
|
"phone number 14155550100"
|
||||||
|
)
|
||||||
|
assert events[("group_created", "2025-08-21 20:13:20.000000")] == (
|
||||||
|
"WhatsApp group chat 'Example Group' "
|
||||||
|
"(120000000000000001@g.us) was created"
|
||||||
|
)
|
||||||
|
assert ("chat_first_message", "2025-08-29 22:40:00.000000") in events
|
||||||
|
assert ("chat_last_message", "2025-08-31 02:26:40.000000") in events
|
||||||
|
|
||||||
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_url_results_includes_expansion():
|
||||||
|
module = Whatsapp(
|
||||||
|
results=[
|
||||||
|
{
|
||||||
|
"links": ["https://bit.ly/message"],
|
||||||
|
"isodate": "2026-07-29 12:00:00.000000",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
module.indicators = Indicators(log=logging.getLogger())
|
||||||
|
module.indicators.resolved_urls["https://bit.ly/message"] = (
|
||||||
|
"https://example.org/landing"
|
||||||
|
)
|
||||||
|
|
||||||
|
module.collect_url_results()
|
||||||
|
|
||||||
|
assert module.url_results == [
|
||||||
|
{
|
||||||
|
"url": "https://bit.ly/message",
|
||||||
|
"expanded_url": "https://example.org/landing",
|
||||||
|
"timestamp": "2026-07-29 12:00:00.000000",
|
||||||
|
"source": "whatsapp",
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# Mobile Verification Toolkit (MVT)
|
||||||
|
# Copyright (c) 2021-2026 The MVT Authors.
|
||||||
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
from mvt.common.module import run_module
|
||||||
|
from mvt.ios.modules.mixed.whatsapp_contacts import WhatsappContacts
|
||||||
|
|
||||||
|
from ..utils import get_ios_backup_folder
|
||||||
|
|
||||||
|
|
||||||
|
class TestWhatsappContactsModule:
|
||||||
|
def test_extraction(self):
|
||||||
|
m = WhatsappContacts(target_path=get_ios_backup_folder())
|
||||||
|
run_module(m)
|
||||||
|
assert len(m.results) == 2
|
||||||
|
|
||||||
|
alice = next(r for r in m.results if r["given_name"] == "Alice")
|
||||||
|
assert alice["full_name"] == "Alice Example"
|
||||||
|
assert alice["phone_number"] == "+14155550100"
|
||||||
|
assert alice["whatsapp_id"] == "14155550100@s.whatsapp.net"
|
||||||
|
assert alice["lid"] == "100000000000001@lid"
|
||||||
|
assert alice["user_name"] == "alice.example"
|
||||||
|
assert alice["disappearing_mode_duration"] == 86400.0
|
||||||
|
assert alice["disappearing_mode_is_on"] is True
|
||||||
|
assert alice["disappearing_mode_label"] == "24 hours"
|
||||||
|
assert alice["disappearing_mode_timestamp"] == "2025-07-23 21:46:40.000000"
|
||||||
|
assert alice["about_timestamp"] == "2025-07-12 08:00:00.000000"
|
||||||
|
assert alice["about_expiration_timestamp"] == "2025-08-16 01:20:00.000000"
|
||||||
|
assert alice["last_updated"] == "2025-08-04 11:33:20.000000"
|
||||||
|
|
||||||
|
bob = next(r for r in m.results if r["given_name"] == "Bob")
|
||||||
|
assert bob["lid"] is None
|
||||||
|
assert bob["disappearing_mode_duration"] is None
|
||||||
|
assert bob["disappearing_mode_is_on"] is False
|
||||||
|
assert bob["disappearing_mode_label"] == "off"
|
||||||
|
assert bob["disappearing_mode_timestamp"] is None
|
||||||
|
|
||||||
|
# Alice: disappearing_mode_set, about_changed, about_expiration and
|
||||||
|
# contact_last_updated. Bob: contact_last_updated only.
|
||||||
|
assert len(m.timeline) == 5
|
||||||
|
|
||||||
|
events = {
|
||||||
|
(entry["event"], entry["timestamp"]): entry["data"]
|
||||||
|
for entry in m.timeline
|
||||||
|
}
|
||||||
|
assert (
|
||||||
|
"24 hours"
|
||||||
|
in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")]
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
"14155550100@s.whatsapp.net (Alice Example)"
|
||||||
|
in events[("disappearing_mode_set", "2025-07-23 21:46:40.000000")]
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
'changed to "Hey there! I am using WhatsApp."'
|
||||||
|
in events[("about_changed", "2025-07-12 08:00:00.000000")]
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
"scheduled to expire"
|
||||||
|
in events[("about_expiration", "2025-08-16 01:20:00.000000")]
|
||||||
|
)
|
||||||
|
|
||||||
|
updated = [
|
||||||
|
entry["data"]
|
||||||
|
for entry in m.timeline
|
||||||
|
if entry["event"] == "contact_last_updated"
|
||||||
|
]
|
||||||
|
assert len(updated) == 2
|
||||||
|
assert all(
|
||||||
|
entry["timestamp"] == "2025-08-04 11:33:20.000000"
|
||||||
|
for entry in m.timeline
|
||||||
|
if entry["event"] == "contact_last_updated"
|
||||||
|
)
|
||||||
|
assert any("14155550101@s.whatsapp.net (Bob Example)" in d for d in updated)
|
||||||
|
|
||||||
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
|
def test_missing_database(self, tmp_path):
|
||||||
|
m = WhatsappContacts(target_path=str(tmp_path))
|
||||||
|
run_module(m)
|
||||||
|
assert m.results == []
|
||||||
|
assert len(m.alertstore.alerts) == 0
|
||||||
@@ -15,8 +15,8 @@ class TestFilesystem:
|
|||||||
def test_filesystem(self):
|
def test_filesystem(self):
|
||||||
m = Filesystem(target_path=get_ios_backup_folder())
|
m = Filesystem(target_path=get_ios_backup_folder())
|
||||||
run_module(m)
|
run_module(m)
|
||||||
assert len(m.results) == 15
|
assert len(m.results) == 23
|
||||||
assert len(m.timeline) == 15
|
assert len(m.timeline) == 23
|
||||||
assert len(m.alertstore.alerts) == 0
|
assert len(m.alertstore.alerts) == 0
|
||||||
|
|
||||||
def test_detection(self, indicator_file):
|
def test_detection(self, indicator_file):
|
||||||
@@ -29,6 +29,6 @@ class TestFilesystem:
|
|||||||
)
|
)
|
||||||
m.indicators = ind
|
m.indicators = ind
|
||||||
run_module(m)
|
run_module(m)
|
||||||
assert len(m.results) == 15
|
assert len(m.results) == 23
|
||||||
assert len(m.timeline) == 15
|
assert len(m.timeline) == 23
|
||||||
assert len(m.alertstore.alerts) == 1
|
assert len(m.alertstore.alerts) == 1
|
||||||
@@ -3,13 +3,19 @@
|
|||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
|
||||||
from click.testing import CliRunner
|
from click.testing import CliRunner
|
||||||
|
|
||||||
from mvt.android.cli import check_androidqf
|
from mvt.android.cli import check_androidqf
|
||||||
|
from mvt.android.cmd_check_androidqf import CmdAndroidCheckAndroidQF
|
||||||
|
from mvt.android.modules.androidqf import ANDROIDQF_MODULES
|
||||||
|
from mvt.android.modules.androidqf.aqf_log_timestamps import AQFLogTimestamps
|
||||||
from mvt.common.config import settings
|
from mvt.common.config import settings
|
||||||
|
|
||||||
from .utils import get_artifact_folder
|
from .utils import get_artifact_folder
|
||||||
@@ -18,12 +24,76 @@ TEST_BACKUP_PASSWORD = "123456"
|
|||||||
|
|
||||||
|
|
||||||
class TestCheckAndroidqfCommand:
|
class TestCheckAndroidqfCommand:
|
||||||
|
def test_log_timestamps_module_is_registered(self):
|
||||||
|
assert AQFLogTimestamps in ANDROIDQF_MODULES
|
||||||
|
|
||||||
def test_check(self):
|
def test_check(self):
|
||||||
runner = CliRunner()
|
runner = CliRunner()
|
||||||
path = os.path.join(get_artifact_folder(), "androidqf")
|
path = os.path.join(get_artifact_folder(), "androidqf")
|
||||||
result = runner.invoke(check_androidqf, [path])
|
result = runner.invoke(check_androidqf, [path])
|
||||||
assert result.exit_code == 0
|
assert result.exit_code == 0
|
||||||
|
|
||||||
|
def test_check_stores_nested_sms_urls(self, tmp_path):
|
||||||
|
runner = CliRunner()
|
||||||
|
path = os.path.join(get_artifact_folder(), "androidqf")
|
||||||
|
|
||||||
|
result = runner.invoke(check_androidqf, ["--output", str(tmp_path), path])
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
urls = json.loads((tmp_path / "urls.json").read_text())
|
||||||
|
assert {entry["url"] for entry in urls} == {
|
||||||
|
"http://google.com",
|
||||||
|
"https://google.com/",
|
||||||
|
}
|
||||||
|
assert all(
|
||||||
|
set(entry) == {"url", "expanded_url", "timestamp", "source"}
|
||||||
|
for entry in urls
|
||||||
|
)
|
||||||
|
assert all(entry["source"] == "sms" for entry in urls)
|
||||||
|
|
||||||
|
def test_acquisition_context_is_passed_to_bugreport(self, tmp_path, mocker):
|
||||||
|
data_path = tmp_path / "androidqf"
|
||||||
|
data_path.mkdir()
|
||||||
|
(data_path / "acquisition.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"started": "2025-06-20T18:00:00Z",
|
||||||
|
"adb_host_public_key": "QUJDRA== acquisition@host",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with zipfile.ZipFile(data_path / "bugreport.zip", "w"):
|
||||||
|
pass
|
||||||
|
|
||||||
|
nested_command = mocker.patch(
|
||||||
|
"mvt.android.cmd_check_androidqf.CmdAndroidCheckBugreport"
|
||||||
|
)
|
||||||
|
nested_command.return_value.timeline = []
|
||||||
|
nested_command.return_value.alertstore.alerts = []
|
||||||
|
command = CmdAndroidCheckAndroidQF(target_path=str(data_path))
|
||||||
|
command.init()
|
||||||
|
|
||||||
|
assert command.run_bugreport_cmd() is True
|
||||||
|
|
||||||
|
assert nested_command.call_args.kwargs["module_options"][
|
||||||
|
"androidqf_acquisition"
|
||||||
|
] == {
|
||||||
|
"started": "2025-06-20T18:00:00Z",
|
||||||
|
"adb_host_public_key": "QUJDRA== acquisition@host",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_acquisition_context_falls_back_to_public_key_file(self, tmp_path):
|
||||||
|
data_path = tmp_path / "androidqf"
|
||||||
|
data_path.mkdir()
|
||||||
|
(data_path / "adb_host_key.pub").write_text("QUJDRA== acquisition@host\n")
|
||||||
|
command = CmdAndroidCheckAndroidQF(target_path=str(data_path))
|
||||||
|
|
||||||
|
command.init()
|
||||||
|
|
||||||
|
assert command.module_options["androidqf_acquisition"] == {
|
||||||
|
"adb_host_public_key": "QUJDRA== acquisition@host\n"
|
||||||
|
}
|
||||||
|
|
||||||
def test_check_encrypted_backup_prompt_valid(self, mocker):
|
def test_check_encrypted_backup_prompt_valid(self, mocker):
|
||||||
"""Prompt for password on CLI"""
|
"""Prompt for password on CLI"""
|
||||||
prompt_mock = mocker.patch(
|
prompt_mock = mocker.patch(
|
||||||
@@ -89,3 +159,25 @@ class TestCheckAndroidqfCommand:
|
|||||||
assert not any(
|
assert not any(
|
||||||
record.levelname in {"CRITICAL", "FATAL"} for record in caplog.records
|
record.levelname in {"CRITICAL", "FATAL"} for record in caplog.records
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_intrusion_log_zip_rejects_path_traversal(self, tmp_path, mocker, caplog):
|
||||||
|
escaped_name = f"mvt-escaped-{tmp_path.name}.txt"
|
||||||
|
escaped_path = os.path.join(tempfile.gettempdir(), escaped_name)
|
||||||
|
archive_path = tmp_path / "androidqf.zip"
|
||||||
|
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||||
|
archive.writestr(f"intrusion_logs/../{escaped_name}", "unsafe")
|
||||||
|
archive.writestr("intrusion_logs/safe.txt", "safe")
|
||||||
|
|
||||||
|
nested_command = mocker.patch(
|
||||||
|
"mvt.android.cmd_check_androidqf.CmdAndroidCheckIntrusionLogs"
|
||||||
|
)
|
||||||
|
nested_command.return_value.timeline = []
|
||||||
|
nested_command.return_value.alertstore.alerts = []
|
||||||
|
command = CmdAndroidCheckAndroidQF(target_path=str(archive_path))
|
||||||
|
command.init()
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
assert command.run_intrusion_logs_cmd() is True
|
||||||
|
|
||||||
|
assert not os.path.exists(escaped_path)
|
||||||
|
assert "Skipping unsafe intrusion log archive entry" in caplog.text
|
||||||
@@ -18,3 +18,13 @@ class TestCheckBugreportCommand:
|
|||||||
path = os.path.join(get_artifact_folder(), "android_data/bugreport/")
|
path = os.path.join(get_artifact_folder(), "android_data/bugreport/")
|
||||||
result = runner.invoke(check_bugreport, [path])
|
result = runner.invoke(check_bugreport, [path])
|
||||||
assert result.exit_code == 0
|
assert result.exit_code == 0
|
||||||
|
|
||||||
|
def test_invalid_zip_reports_clean_error(self, tmp_path):
|
||||||
|
path = tmp_path / "invalid.zip"
|
||||||
|
path.write_bytes(b"not a zip archive")
|
||||||
|
|
||||||
|
result = CliRunner().invoke(check_bugreport, [str(path)])
|
||||||
|
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "Invalid bugreport archive" in result.output
|
||||||
|
assert "Traceback" not in result.output
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
from mvt.ios.cli import check_sysdiagnose
|
||||||
|
|
||||||
|
|
||||||
|
CUSTOM_MODULE = """
|
||||||
|
from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
|
||||||
|
|
||||||
|
|
||||||
|
class CustomSysdiagnoseModule(SysdiagnoseExtraction):
|
||||||
|
supported_commands = (("ios", "check-sysdiagnose"),)
|
||||||
|
slug = "custom_sysdiagnose_module"
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
file_path = self._get_files_by_pattern("*/artifact.txt")[0]
|
||||||
|
self.results = [{"content": self._get_file_content(file_path).decode("utf-8")}]
|
||||||
|
|
||||||
|
def check_indicators(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def serialize(self, result):
|
||||||
|
return None
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _create_sysdiagnose_folder(tmp_path):
|
||||||
|
folder = tmp_path / "sysdiagnose"
|
||||||
|
folder.mkdir()
|
||||||
|
(folder / "artifact.txt").write_text("artifact", encoding="utf-8")
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_sysdiagnose_runs_explicitly_scoped_custom_module(tmp_path):
|
||||||
|
module_path = tmp_path / "custom_sysdiagnose.py"
|
||||||
|
module_path.write_text(CUSTOM_MODULE, encoding="utf-8")
|
||||||
|
output_path = tmp_path / "output"
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
check_sysdiagnose,
|
||||||
|
[
|
||||||
|
"--load-module",
|
||||||
|
str(module_path),
|
||||||
|
"--output",
|
||||||
|
str(output_path),
|
||||||
|
str(_create_sysdiagnose_folder(tmp_path)),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert (output_path / "custom_sysdiagnose_module.json").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_sysdiagnose_requires_an_explicitly_scoped_module(tmp_path):
|
||||||
|
result = CliRunner().invoke(check_sysdiagnose, [str(_create_sysdiagnose_folder(tmp_path))])
|
||||||
|
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "No custom modules support mvt-ios check-sysdiagnose" in result.output
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
import io
|
||||||
|
import tarfile
|
||||||
|
from datetime import timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from mvt.ios.cmd_check_sysdiagnose import CmdIOSCheckSysdiagnose
|
||||||
|
from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
|
||||||
|
|
||||||
|
|
||||||
|
class SysdiagnoseTestModule(SysdiagnoseExtraction):
|
||||||
|
supported_commands = (("ios", "check-sysdiagnose"),)
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
file_path = self._get_files_by_pattern("*/artifact.txt")[0]
|
||||||
|
self.results = [
|
||||||
|
{
|
||||||
|
"content": self._get_file_content(file_path).decode("utf-8"),
|
||||||
|
"timezone_offset": self._extract_timezone().utcoffset(None).seconds,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def check_indicators(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def serialize(self, result):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _create_sysdiagnose_folder(tmp_path):
|
||||||
|
folder = tmp_path / "sysdiagnose"
|
||||||
|
folder.mkdir()
|
||||||
|
(folder / "artifact.txt").write_text("artifact", encoding="utf-8")
|
||||||
|
(folder / "sysdiagnose.log").write_text(
|
||||||
|
"sysdiagnose_2024.01.02_03-04-05+0200.tar.gz", encoding="utf-8"
|
||||||
|
)
|
||||||
|
(folder / "report.ips").write_text('{"bug_type": 210}\nbody', encoding="utf-8")
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
|
def _create_sysdiagnose_archive(tmp_path, folder):
|
||||||
|
archive_path = tmp_path / "sysdiagnose.tar.gz"
|
||||||
|
with tarfile.open(archive_path, "w:gz") as archive:
|
||||||
|
for path in folder.iterdir():
|
||||||
|
archive.add(path, arcname=f"sysdiagnose/{path.name}")
|
||||||
|
return archive_path
|
||||||
|
|
||||||
|
|
||||||
|
def _run_command(path):
|
||||||
|
command = CmdIOSCheckSysdiagnose(
|
||||||
|
target_path=str(path), custom_modules=[SysdiagnoseTestModule]
|
||||||
|
)
|
||||||
|
command.run()
|
||||||
|
return command
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_sysdiagnose_from_folder(tmp_path):
|
||||||
|
command = _run_command(_create_sysdiagnose_folder(tmp_path))
|
||||||
|
|
||||||
|
assert command.executed[0].results == [
|
||||||
|
{"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
|
||||||
|
]
|
||||||
|
assert command.executed[0].ips_files == [
|
||||||
|
{"file_path": str(tmp_path / "sysdiagnose" / "report.ips"), "bug_type": 210}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_sysdiagnose_from_archive_closes_archive(tmp_path):
|
||||||
|
folder = _create_sysdiagnose_folder(tmp_path)
|
||||||
|
command = _run_command(_create_sysdiagnose_archive(tmp_path, folder))
|
||||||
|
|
||||||
|
assert command.executed[0].results == [
|
||||||
|
{"content": "artifact", "timezone_offset": timedelta(hours=2).seconds}
|
||||||
|
]
|
||||||
|
assert command.executed[0].ips_files == [
|
||||||
|
{
|
||||||
|
"file_path": str(
|
||||||
|
Path(command.extracted_sysdiagnose_path) / "report.ips"
|
||||||
|
),
|
||||||
|
"bug_type": 210,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
assert command.sysdiagnose_archive is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_archive_is_extracted_once_and_unsafe_members_are_skipped(tmp_path):
|
||||||
|
archive_path = tmp_path / "sysdiagnose.tar.gz"
|
||||||
|
escaped_path = tmp_path / "escaped.txt"
|
||||||
|
content = b"test content"
|
||||||
|
member = tarfile.TarInfo("sysdiagnose/artifact.txt")
|
||||||
|
member.size = len(content)
|
||||||
|
|
||||||
|
with tarfile.open(archive_path, "w:gz") as archive:
|
||||||
|
archive.addfile(member, io.BytesIO(content))
|
||||||
|
escaped = tarfile.TarInfo(f"sysdiagnose/../../{escaped_path.name}")
|
||||||
|
escaped.size = len(content)
|
||||||
|
archive.addfile(escaped, io.BytesIO(content))
|
||||||
|
link = tarfile.TarInfo("sysdiagnose/link")
|
||||||
|
link.type = tarfile.SYMTYPE
|
||||||
|
link.linkname = "/etc/hostname"
|
||||||
|
archive.addfile(link)
|
||||||
|
|
||||||
|
command = CmdIOSCheckSysdiagnose(target_path=str(archive_path))
|
||||||
|
try:
|
||||||
|
command.init()
|
||||||
|
extracted_path = Path(command.extracted_sysdiagnose_path)
|
||||||
|
assert (extracted_path / "artifact.txt").read_bytes() == content
|
||||||
|
assert not escaped_path.exists()
|
||||||
|
assert not (extracted_path / "link").exists()
|
||||||
|
|
||||||
|
module = SysdiagnoseExtraction()
|
||||||
|
command.module_init(module)
|
||||||
|
assert module.tar is None
|
||||||
|
assert module.parent_path == str(extracted_path.parent)
|
||||||
|
finally:
|
||||||
|
command.finish()
|
||||||
|
|
||||||
|
assert not extracted_path.exists()
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import sqlite3
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
@@ -37,6 +38,22 @@ def get_indicator_file():
|
|||||||
print("PYTEST env", os.getenv("PYTEST_CURRENT_TEST"))
|
print("PYTEST env", os.getenv("PYTEST_CURRENT_TEST"))
|
||||||
|
|
||||||
|
|
||||||
|
def add_backup_manifest_entry(backup_path, file_id, domain, relative_path):
|
||||||
|
"""
|
||||||
|
Register an extra file in a test backup's Manifest.db
|
||||||
|
"""
|
||||||
|
conn = sqlite3.connect(os.path.join(backup_path, "Manifest.db"))
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO Files (fileID, domain, relativePath, flags, file) "
|
||||||
|
"VALUES (?, ?, ?, 1, ?);",
|
||||||
|
(file_id, domain, relative_path, b""),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
# Checkpoint the test change so the fixture does not retain SQLite sidecars.
|
||||||
|
conn.execute("PRAGMA journal_mode=DELETE;")
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
def delete_tmp_db_files(file_path):
|
def delete_tmp_db_files(file_path):
|
||||||
"""
|
"""
|
||||||
Remove Sqlite temporary files that appear on some platforms
|
Remove Sqlite temporary files that appear on some platforms
|
||||||
|
|||||||
Reference in new issue
Block a user