Skip modules with unavailable dependencies instead of aborting the run

A module declaring a dependency its command does not provide made
_ordered_modules() give up on the whole run, so a single wrong declaration
in a module scoped to several commands turned a forensic analysis into
zero executed modules with one warning to explain it.

Drop only the modules that cannot run: the one with the unavailable
dependency, and anything depending on it. Each gets its own warning naming
the module missing a dependency and the dependency it is missing, and the
remaining modules run in the same stable topological order as before. A
cycle in the dependency graph is still a programming error and still stops
the run.

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "uv" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7

.github/workflows/add-issue-to-project.yml

@@ -0,0 +1,19 @@
+name: Add issue to project
+
+on:
+  issues:
+    types:
+      - opened
+      - reopened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v2
+        with:
+          # You can target a project in a different organization
+          # to the issue
+          project-url: https://github.com/orgs/mvt-project/projects/1
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}

.github/workflows/mypy.yml

@@ -0,0 +1,28 @@
+name: Mypy
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  mypy_py3:
+    name: Mypy check
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+      - name: Setup Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.10"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v10.0.0
+        with:
+          enable-cache: true
+      - name: Install Dependencies
+        run: |
+          uv sync --locked --group dev
+      - name: mypy
+        run: |
+          make mypy

.github/workflows/publish-release-docker.yml

@@ -0,0 +1,82 @@
+#
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a release is published.
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  release:
+    types: [published]
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+      #
+    strategy:
+      matrix:
+        platform:
+          - dockerfile: "Dockerfile"
+            tag-suffix: ""
+          - dockerfile: "Dockerfile.ios"
+            tag-suffix: "-ios"
+          - dockerfile: "Dockerfile.android"
+            tag-suffix: "-android"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=latest,enable={{ is_default_branch }},suffix=${{ matrix.platform.tag-suffix }}
+            type=raw,enable=${{ github.event_name == 'release' || github.ref_type == 'tag' }},value=stable,suffix=${{ matrix.platform.tag-suffix }}
+            type=raw,enable=${{ github.event_name == 'release' }},value=${{ github.event.release.tag_name }},suffix=${{ matrix.platform.tag-suffix }}
+            type=raw,enable=${{ github.ref_type == 'tag' }},value=${{ github.ref_name }},suffix=${{ matrix.platform.tag-suffix }}
+            type=sha,suffix=${{ matrix.platform.tag-suffix }}
+            type=sha,format=long,suffix=${{ matrix.platform.tag-suffix }}
+      # Register emulators so Buildx can build the ARM64 images on the AMD64 runner.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      # This step sets up some additional capabilities to generate the provenance and sbom attestations
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v7
+        with:
+          file: ${{ matrix.platform.dockerfile }}
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          provenance: mode=max
+          sbom: true

.github/workflows/ruff.yml

@@ -0,0 +1,29 @@
+name: Ruff
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  ruff_py3:
+    name: Ruff syntax check
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+      - name: Setup Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.10"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v10.0.0
+        with:
+          enable-cache: true
+      - name: Install Dependencies
+        run: |
+          uv sync --locked --group dev
+      - name: ruff
+        run: |
+          make ruff

.github/workflows/scripts/update-ios-releases.py

@@ -0,0 +1,96 @@
+"""
+Python script to download the Apple RSS feed and parse it.
+"""
+
+import json
+import os
+import urllib.request
+from xml.dom.minidom import parseString
+
+from packaging import version
+
+
+def download_apple_rss(feed_url):
+    with urllib.request.urlopen(feed_url) as f:
+        rss_feed = f.read().decode("utf-8")
+    print("Downloaded RSS feed from Apple.")
+    return rss_feed
+
+
+def parse_latest_ios_versions(rss_feed_text):
+    latest_ios_versions = []
+
+    parsed_feed = parseString(rss_feed_text)
+    for item in parsed_feed.getElementsByTagName("item"):
+        title = item.getElementsByTagName("title")[0].firstChild.data
+        if not title.startswith("iOS"):
+            continue
+
+        import re
+
+        build_match = re.match(
+            r"iOS (?P<version>[\d\.]+) (?P<beta>beta )?(\S*)?\((?P<build>.*)\)", title
+        )
+        if not build_match:
+            print("Could not parse iOS build:", title)
+            continue
+
+        # Handle iOS beta releases
+        release_info = build_match.groupdict()
+        release_beta = release_info.pop("beta")
+        if release_beta:
+            print("Skipping beta release:", title)
+            continue
+
+        # Some iOS releases have multiple build number for different hardware models.
+        # We will split these into separate entries and record each build number.
+        build_list = release_info.pop("build")
+        build_variants = build_list.split(" | ")
+        for build_number in build_variants:
+            release_info["build"] = build_number
+            latest_ios_versions.append(release_info)
+
+    return latest_ios_versions
+
+
+def update_mvt(mvt_checkout_path, latest_ios_versions):
+    version_path = os.path.join(mvt_checkout_path, "src/mvt/ios/data/ios_versions.json")
+    with open(version_path, "r") as version_file:
+        current_versions = json.load(version_file)
+
+    new_entry_count = 0
+    for new_version in latest_ios_versions:
+        for current_version in current_versions:
+            if new_version["build"] == current_version["build"]:
+                break
+        else:
+            # New version that does not exist in current data
+            current_versions.append(new_version)
+            new_entry_count += 1
+
+    if not new_entry_count:
+        print("No new iOS versions found.")
+    else:
+        print("Found {} new iOS versions.".format(new_entry_count))
+        new_version_list = sorted(
+            current_versions, key=lambda x: version.Version(x["version"])
+        )
+        with open(version_path, "w") as version_file:
+            json.dump(new_version_list, version_file, indent=4)
+
+
+def main():
+    print("Downloading RSS feed...")
+    mvt_checkout_path = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "../../../")
+    )
+
+    rss_feed = download_apple_rss(
+        "https://developer.apple.com/news/releases/rss/releases.rss"
+    )
+    latest_ios_version = parse_latest_ios_versions(rss_feed)
+    update_mvt(mvt_checkout_path, latest_ios_version)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/tests.yml

@@ -0,0 +1,41 @@
+name: Tests
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    name: Run Python Tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Install uv
+        uses: astral-sh/setup-uv@v10.0.0
+        with:
+          enable-cache: true
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python dependencies
+        run: |
+          uv sync --locked --group dev --python ${{ matrix.python-version }}
+      - name: Test with pytest
+        run: |
+          set -o pipefail
+          make test-ci | tee pytest-coverage.txt
+
+      - name: Pytest coverage comment
+        continue-on-error: true # Workflows running on a fork can't post comments
+        uses: MishaKav/pytest-coverage-comment@main
+        if: github.event_name == 'pull_request'
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml

.github/workflows/update-ios-data.yml

@@ -0,0 +1,32 @@
+name: Update iOS releases and version numbers
+run-name: ${{ github.actor }} is finding the latest iOS release version and build numbers
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '0 */6 * * *'
+
+
+jobs:
+  update-ios-version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+      - name: Install script dependencies
+        run: python -m pip install "packaging==26.3"
+      - name: Run script to fetch latest iOS releases from Apple RSS feed.
+        run: python .github/workflows/scripts/update-ios-releases.py
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v8
+        with:
+          title: '[auto] Update iOS releases and versions'
+          commit-message: Add new iOS versions and build numbers
+          branch: auto/add-new-ios-releases
+          draft: true
+          body: |
+            This is an automated pull request to update the iOS releases and version numbers.
+          add-paths: |
+            *.json
+          labels: |
+            automated pr

.gitignore

@@ -50,6 +50,8 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+pytest-coverage.txt
+pytest.xml
 
 # Translations
 *.mo
@@ -84,6 +86,9 @@ ipython_config.py
 # pyenv
 .python-version
 
+# uv project Python version
+!.python-version
+
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
@@ -129,3 +134,11 @@ dmypy.json
 .pyre/
 *.pyc
 
+# Temporal files
+*~
+
+# IDEA Dev Environment
+.idea
+
+# Sublime Text project files
+*.sublime*

.python-version

@@ -0,0 +1 @@
+3.10

.readthedocs.yaml

@@ -5,11 +5,17 @@
 # Required
 version: 2
 
+build:
+  os: "ubuntu-24.04"
+  tools:
+    python: "3.11"
+
 mkdocs:
   configuration: mkdocs.yml
 
-# Optionally set the version of Python and requirements required to build your docs
 python:
-   version: 3.7
-   install:
-   - requirements: docs/requirements.txt
+  install:
+    - method: uv
+      command: sync
+      groups:
+        - docs

.safety-policy.yml

@@ -0,0 +1,11 @@
+# Safety Security and License Configuration file
+# We recommend checking this file into your source control in the root of your Python project
+# If this file is named .safety-policy.yml and is in the same directory where you run `safety check` it will be used by default.
+# Otherwise, you can use the flag `safety check --policy-file <path-to-this-file>` to specify a custom location and name for the file.
+# To validate and review your policy file, run the validate command: `safety validate policy_file --path <path-to-this-file>`
+security: # configuration for the `safety check` command
+    ignore-vulnerabilities: # Here you can list multiple specific vulnerabilities you want to ignore (optionally for a time period)
+        67599: # Example vulnerability ID
+            reason: disputed, inapplicable
+        70612:
+            reason: disputed, inapplicable

CONTRIBUTING.md

@@ -0,0 +1,65 @@
+# Contributing to Mobile Verification Toolkit (MVT)
+
+We greatly appreciate contributions to MVT!
+
+Your involvement, whether through identifying issues, improving functionality, or enhancing documentation, is very much appreciated. To ensure smooth collaboration and a welcoming environment, we've outlined some key guidelines for contributing below.
+
+## Getting started
+
+Contributing to an open-source project like MVT might seem overwhelming at first, but we're here to support you!
+
+ Whether you're a technologist, a frontline human rights defender, a field researcher, or someone new to consensual spyware forensics, there are many ways to make meaningful contributions.
+
+ Here's how you can get started:
+
+1. **Explore the codebase:**
+    - Browse the repository to get familar with MVT. Many MVT modules are simple in functionality and easy to understand.
+    - Look for `TODO:` or `FIXME:` comments in the code for areas that need attention.
+
+2. **Check Github issues:**
+    - Look for issues tagged with ["help wanted"](https://github.com/mvt-project/mvt/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) or ["good first issue"](https://github.com/mvt-project/mvt/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) to find tasks that are beginner-friendly or where input from the community would be helpful.
+
+3. **Ask for guidance:**
+
+    - If you're unsure where to start, feel free to open a [discussion](https://github.com/mvt-project/mvt/discussions) or comment on an issue.
+
+## How to contribute:
+
+1. **Report issues:**
+
+ - Found a bug? Please check existing issues to see if it's already reported. If not, open a new issue. Mobile operating systems and databases are constantly evolving, an new errors may appear spontaniously in new app versions.
+
+ **Please provide as much information as possible about the prodblem including: any error messages, steps to reproduce the problem, and any logs or screenshots that can help.**
+
+
+2. **Suggest features:**
+    - If you have an idea for new functionality, create a feature request issue and describe your proposal.
+
+3. **Submit code:**
+    - Fork the repository and create a new branch for your changes.
+    - Ensure your changes align with the code style guidelines (see below).
+    - Open a pull request (PR) with a clear description of your changes and link it to any relevant issues.
+
+4. **Documentation contributions:**
+    - Improving documentation is just as valuable as contributing code! If you notice gaps or inaccuracies in the documentation, feel free to submit changes or suggest updates.
+
+## Code style
+Please follow these code style guidelines for consistency and readability:
+
+- **Indentation**: use 4 spaces per tab.
+- **Quotes**: Use double quotes (`"`) by default. Use single quotes (`'`) for nested strings instead of escaping (`\"`), or when using f-formatting.
+- **Maximum line length**:
+    - Aim for lines no longer than 80 characters.
+    - Exceptions are allowed for long log lines or strings, which may extend up to 100 characters.
+    - Wrap lines that exceed 100 characters.
+
+Follow [PEP 8 guidelines](https://peps.python.org/pep-0008/) for indentation and overall Python code style. All MVT code is automatically linted with [Ruff](https://github.com/astral-sh/ruff) before merging.
+
+Please check your code before opening a pull request by running `make ruff`
+
+
+## Community and support
+
+We aim to create a supportive and collaborative environment for all contributors. If you run into any challenges, feel free to reach out through the discussions or issues section of the repository.
+
+Your contributions, big or small, help improve MVT and are always appreciated.

Dockerfile

@@ -0,0 +1,167 @@
+# Base image for building libraries
+# ---------------------------------
+FROM ubuntu:22.04 AS build-base
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install build tools and dependencies
+RUN apt-get update \
+  && apt-get install -y \
+  build-essential \
+  git \
+  autoconf \
+  automake \
+  libtool-bin \
+  pkg-config \
+  libcurl4-openssl-dev \
+  libusb-1.0-0-dev \
+  libssl-dev \
+  udev \
+  && rm -rf /var/lib/apt/lists/*
+
+
+# libplist
+# --------
+FROM build-base AS build-libplist
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libplist && cd libplist \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libplist
+
+
+# libimobiledevice-glue
+# ---------------------
+FROM build-base AS build-libimobiledevice-glue
+
+# Install dependencies
+COPY --from=build-libplist /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libimobiledevice-glue && cd libimobiledevice-glue \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libimobiledevice-glue
+
+
+# libtatsu
+# --------
+FROM build-base AS build-libtatsu
+
+# Install dependencies
+COPY --from=build-libplist /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libtatsu && cd libtatsu \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libtatsu
+
+
+# libusbmuxd
+# ----------
+FROM build-base AS build-libusbmuxd
+
+# Install dependencies
+COPY --from=build-libplist /build /
+COPY --from=build-libimobiledevice-glue /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libusbmuxd && cd libusbmuxd \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libusbmuxd
+
+
+# libimobiledevice
+# ----------------
+FROM build-base AS build-libimobiledevice
+
+# Install dependencies
+COPY --from=build-libplist /build /
+COPY --from=build-libtatsu /build /
+COPY --from=build-libimobiledevice-glue /build /
+COPY --from=build-libusbmuxd /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libimobiledevice && cd libimobiledevice \
+  && ./autogen.sh --enable-debug && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libimobiledevice
+
+
+# usbmuxd
+# -------
+FROM build-base AS build-usbmuxd
+
+# Install dependencies
+COPY --from=build-libplist /build /
+COPY --from=build-libimobiledevice-glue /build /
+COPY --from=build-libusbmuxd /build /
+COPY --from=build-libimobiledevice /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
+  && ./autogen.sh --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf usbmuxd && mv /build/lib /build/usr/lib
+
+
+# Create main image
+FROM ubuntu:24.04 AS main
+COPY --from=ghcr.io/astral-sh/uv:0.11.8 /uv /uvx /usr/local/bin/
+
+LABEL org.opencontainers.image.url="https://mvt.re"
+LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
+LABEL org.opencontainers.image.source="https://github.com/mvt-project/mvt"
+LABEL org.opencontainers.image.title="Mobile Verification Toolkit"
+LABEL org.opencontainers.image.description="MVT is a forensic tool to look for signs of infection in smartphone devices."
+LABEL org.opencontainers.image.licenses="MVT License 1.1"
+LABEL org.opencontainers.image.base.name=docker.io/library/ubuntu:22.04
+
+# Install runtime dependencies
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+  && apt-get install -y \
+  adb \
+  binutils \
+  default-jre-headless \
+  file \
+  jq \
+  less \
+  libcurl4 \
+  libimage-exiftool-perl \
+  libssl3 \
+  libusb-1.0-0 \
+  moreutils \
+  p7zip-full \
+  python3 \
+  ripgrep \
+  sqlite3 \
+  tree \
+  unzip \
+  xxd
+COPY --from=build-libplist /build /
+COPY --from=build-libimobiledevice-glue /build /
+COPY --from=build-libtatsu /build /
+COPY --from=build-libusbmuxd /build /
+COPY --from=build-libimobiledevice /build /
+COPY --from=build-usbmuxd /build /
+
+# Install mvt using the locally checked out source
+COPY . mvt/
+RUN uv pip install --system --break-system-packages --no-cache ./mvt \
+  && rm -rf mvt
+
+# Installing ABE
+ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2e11632 \
+  https://github.com/nelenkov/android-backup-extractor/releases/download/master-20221109063121-8fdfc5e/abe.jar /opt/abe/abe.jar
+# Create alias for abe
+RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+
+# Generate adb key folder
+RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc
+RUN mkdir /root/.android
+
+# Setup investigations environment
+RUN mkdir /home/cases
+WORKDIR /home/cases
+RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc \
+  && echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
+
+CMD /bin/bash

Dockerfile.android

@@ -0,0 +1,37 @@
+# Create main image
+FROM python:3.10.14-alpine3.20 AS main
+COPY --from=ghcr.io/astral-sh/uv:0.11.8 /uv /uvx /usr/local/bin/
+
+LABEL org.opencontainers.image.url="https://mvt.re"
+LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
+LABEL org.opencontainers.image.source="https://github.com/mvt-project/mvt"
+LABEL org.opencontainers.image.title="Mobile Verification Toolkit (Android)"
+LABEL org.opencontainers.image.description="MVT is a forensic tool to look for signs of infection in smartphone devices."
+LABEL org.opencontainers.image.licenses="MVT License 1.1"
+LABEL org.opencontainers.image.base.name=docker.io/library/python:3.10.14-alpine3.20
+
+# Install runtime dependencies
+RUN apk add --no-cache \
+  android-tools \
+  git \
+  libusb \
+  openjdk11-jre-headless \
+  sqlite
+
+# Install mvt
+COPY ./ mvt
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev \
+  && uv pip install --system --no-cache ./mvt \
+  && apk del .build-deps gcc musl-dev && rm -rf ./mvt
+
+# Installing ABE
+ADD --checksum=sha256:a20e07f8b2ea47620aff0267f230c3f1f495f097081fd709eec51cf2a2e11632 \
+  https://github.com/nelenkov/android-backup-extractor/releases/download/master-20221109063121-8fdfc5e/abe.jar /opt/abe/abe.jar
+# Create alias for abe
+RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+
+# Generate adb key folder
+RUN echo 'if [ ! -f /root/.android/adbkey ]; then adb keygen /root/.android/adbkey > /dev/null 2>&1; fi' >> ~/.bashrc
+RUN mkdir /root/.android
+
+ENTRYPOINT [ "/usr/local/bin/mvt-android" ]

Dockerfile.ios

@@ -0,0 +1,138 @@
+# Base image for building libraries
+# ---------------------------------
+FROM ubuntu:22.04 AS build-base
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install build tools and dependencies
+RUN apt-get update \
+  && apt-get install -y \
+  build-essential \
+  git \
+  autoconf \
+  automake \
+  libtool-bin \
+  pkg-config \
+  libcurl4-openssl-dev \
+  libusb-1.0-0-dev \
+  libssl-dev \
+  udev \
+  && rm -rf /var/lib/apt/lists/*
+
+
+# libplist
+# --------
+FROM build-base AS build-libplist
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libplist && cd libplist \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libplist
+
+
+# libimobiledevice-glue
+# ---------------------
+FROM build-base AS build-libimobiledevice-glue
+
+# Install dependencies
+COPY --from=build-libplist /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libimobiledevice-glue && cd libimobiledevice-glue \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libimobiledevice-glue
+
+
+# libtatsu
+# --------
+FROM build-base AS build-libtatsu
+
+# Install dependencies
+COPY --from=build-libplist /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libtatsu && cd libtatsu \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libtatsu
+
+
+# libusbmuxd
+# ----------
+FROM build-base AS build-libusbmuxd
+
+# Install dependencies
+COPY --from=build-libplist /build /
+COPY --from=build-libimobiledevice-glue /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libusbmuxd && cd libusbmuxd \
+  && ./autogen.sh && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libusbmuxd
+
+
+# libimobiledevice
+# ----------------
+FROM build-base AS build-libimobiledevice
+
+# Install dependencies
+COPY --from=build-libplist /build /
+COPY --from=build-libtatsu /build /
+COPY --from=build-libimobiledevice-glue /build /
+COPY --from=build-libusbmuxd /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/libimobiledevice && cd libimobiledevice \
+  && ./autogen.sh --enable-debug && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf libimobiledevice
+
+
+# usbmuxd
+# -------
+FROM build-base AS build-usbmuxd
+
+# Install dependencies
+COPY --from=build-libplist /build /
+COPY --from=build-libimobiledevice-glue /build /
+COPY --from=build-libusbmuxd /build /
+COPY --from=build-libimobiledevice /build /
+
+# Build
+RUN git clone https://github.com/libimobiledevice/usbmuxd && cd usbmuxd \
+  && ./autogen.sh --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make -j "$(nproc)" && make install DESTDIR=/build \
+  && cd .. && rm -rf usbmuxd && mv /build/lib /build/usr/lib
+
+
+# Main image
+# ----------
+FROM python:3.10.14-alpine3.20 AS main
+COPY --from=ghcr.io/astral-sh/uv:0.11.8 /uv /uvx /usr/local/bin/
+
+LABEL org.opencontainers.image.url="https://mvt.re"
+LABEL org.opencontainers.image.documentation="https://docs.mvt.re"
+LABEL org.opencontainers.image.source="https://github.com/mvt-project/mvt"
+LABEL org.opencontainers.image.title="Mobile Verification Toolkit (iOS)"
+LABEL org.opencontainers.image.description="MVT is a forensic tool to look for signs of infection in smartphone devices."
+LABEL org.opencontainers.image.licenses="MVT License 1.1"
+LABEL org.opencontainers.image.base.name=docker.io/library/python:3.10.14-alpine3.20
+
+# Install runtime dependencies
+RUN apk add --no-cache \
+  gcompat \
+  libcurl \
+  libssl3 \
+  libusb \
+  sqlite
+COPY --from=build-libplist /build /
+COPY --from=build-libimobiledevice-glue /build /
+COPY --from=build-libtatsu /build /
+COPY --from=build-libusbmuxd /build /
+COPY --from=build-libimobiledevice /build /
+COPY --from=build-usbmuxd /build /
+
+# Install mvt using the locally checked out source
+COPY ./ mvt
+RUN apk add --no-cache --virtual .build-deps git gcc musl-dev \
+  && uv pip install --system --no-cache ./mvt \
+  && apk del .build-deps git gcc musl-dev && rm -rf ./mvt
+
+ENTRYPOINT [ "/usr/local/bin/mvt-ios" ]

LICENSE

@@ -1,4 +1,4 @@
-MVT License 1.0
+MVT License 1.1
 ===============
 
 1. Definitions
@@ -35,7 +35,7 @@ MVT License 1.0
     means any form of the work other than Source Code Form.
 
 1.7. "Larger Work"
-    means a work that combines Covered Software with other material, in 
+    means a work that combines Covered Software with other material, in
     a separate file or files, that is not Covered Software.
 
 1.8. "License"
@@ -89,16 +89,16 @@ MVT License 1.0
     a Larger Work.
 
 1.16. "Device Owner" (or "Device Owners")
-    means an individal or a legal entity with legal ownership of an
+    means an individual or a legal entity with legal ownership of an
     electronic device which is being analysed through the use of
     Covered Software or a Larger Work, or from which Data was extracted
     for subsequent analysis.
 
 1.17. "Data Owner" (or "Data Owners")
-    means an individial or group of individuals who made use of the
-    electronic device from which Data that is extracted and/or analyzed
-    originated. "Data Owner" might or might not differ from "Device
-    Owner".
+    means an individual or group of individuals who made legitimate use
+    of the electronic device from which Data that is extracted and/or
+    analyzed originated. "Data Owner" might or might not differ from
+    "Device Owner".
 
 2. License Grants and Conditions
 --------------------------------
@@ -381,8 +381,8 @@ Exhibit A - Source Code Form License Notice
 -------------------------------------------
 
   This Source Code Form is subject to the terms of the MVT License,
-  v. 1.0. If a copy of the MVT License was not distributed with this
-  file, You can obtain one at TODO.
+  v. 1.1. If a copy of the MVT License was not distributed with this
+  file, You can obtain one at https://license.mvt.re/1.1/.
 
 If it is not possible or desirable to put the notice in a particular
 file, then You may include the notice in a location (such as a LICENSE
@@ -395,7 +395,7 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 ---------------------------------------------------------
 
   This Source Code Form is "Incompatible With Secondary Licenses", as
-  defined by the MVT License, v. 1.0.
+  defined by the MVT License, v. 1.1.
 
 
 This license is an adaption of Mozilla Public License, v. 2.0.

Makefile

@@ -1,10 +1,39 @@
 PWD = $(shell pwd)
+UV ?= uv
+
+check: ruff mypy
+
+ruff:
+	$(UV) run ruff check .
+
+mypy:
+	$(UV) run mypy
+
+test:
+	$(UV) run pytest
+
+test-ci:
+	$(UV) run pytest -v
+
+install:
+	$(UV) sync
+
+test-requirements:
+	$(UV) sync --group dev
+
+generate-proto-parsers:
+	# Generate python parsers for protobuf files
+	PROTO_FILES=$$(find src/mvt/android/parsers/proto/ -iname "*.proto"); \
+	$(UV) run protoc -Isrc/mvt/android/parsers/proto/ --python_betterproto2_out=src/mvt/android/parsers/proto/ $$PROTO_FILES
 
 clean:
-	rm -rf $(PWD)/build $(PWD)/dist $(PWD)/mvt.egg-info
+	rm -rf $(PWD)/build $(PWD)/dist $(PWD)/src/mvt.egg-info
 
 dist:
-	python3 setup.py sdist bdist_wheel
+	$(UV) build
 
 upload:
-	python3 -m twine upload dist/*
+	$(UV) tool run twine upload dist/*
+
+test-upload:
+	$(UV) tool run twine upload --repository testpypi dist/*

README.md

@@ -1,45 +1,93 @@
 <p align="center">
-     <img src="./docs/mvt.png" width="300" />
+     <img src="https://docs.mvt.re/en/latest/mvt.png" width="200" />
 </p>
 
 # Mobile Verification Toolkit
 
+> [!IMPORTANT]
+> We recently merged the "v3" branch. This introduced breaking changes. If you relied on mvt output in other scripts They might have broken. More details: https://github.com/mvt-project/mvt/issues/757
+
 [![](https://img.shields.io/pypi/v/mvt)](https://pypi.org/project/mvt/)
+[![Documentation Status](https://readthedocs.org/projects/mvt/badge/?version=latest)](https://docs.mvt.re/en/latest/?badge=latest)
+[![CI](https://github.com/mvt-project/mvt/actions/workflows/tests.yml/badge.svg)](https://github.com/mvt-project/mvt/actions/workflows/tests.yml)
+[![Downloads](https://pepy.tech/badge/mvt)](https://pepy.tech/project/mvt)
 
 Mobile Verification Toolkit (MVT) is a collection of utilities to simplify and automate the process of gathering forensic traces helpful to identify a potential compromise of Android and iOS devices.
 
-[Please check out the documentation.](https://mvt.readthedocs.io/en/latest/)
+It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors.
+
+> **Note**
+> MVT is a forensic research tool intended for technologists and investigators. It requires understanding digital forensics and using command-line tools. This is not intended for end-user self-assessment. If you are concerned with the security of your device please seek reputable expert assistance.
+>
+
+### Indicators of Compromise
+
+MVT supports using public [indicators of compromise (IOCs)](https://github.com/mvt-project/mvt-indicators) to scan mobile devices for potential traces of targeting or infection by known spyware campaigns. This includes IOCs published by [Amnesty International](https://github.com/AmnestyTech/investigations/) and other  research groups.
+
+> **Warning**
+> Public indicators of compromise are insufficient to determine that a device is "clean", and not targeted with a particular spyware tool. Reliance on public indicators alone can miss recent forensic traces and give a false sense of security.
+>
+> Reliable and comprehensive digital forensic support and triage requires access to non-public indicators, research and threat intelligence.
+>
+>Such support is available to civil society through [Amnesty International's Security Lab](https://securitylab.amnesty.org/get-help/?c=mvt_docs) or through our forensic partnership with [Access Now’s Digital Security Helpline](https://www.accessnow.org/help/).
+
+More information about using indicators of compromise with MVT is available in the [documentation](https://docs.mvt.re/en/latest/iocs/).
 
 ## Installation
 
-First you need to install dependencies, on Linux `sudo apt install python3 python3-pip libusb-1.0-0` or on MacOS `brew install python3 libusb`.
+MVT can be installed from sources or from [PyPI](https://pypi.org/project/mvt/) (you will need some dependencies, check the [documentation](https://docs.mvt.re/en/latest/install/)):
 
-Then you can install mvt from pypi with `pip install mvt`, or directly form sources:
-```bash
-git clone https://github.com/mvt-project/mvt.git
-cd mvt
-pip3 install .
 ```
+pip3 install mvt
+```
+
+You can also install MVT from PyPI with [uv](https://docs.astral.sh/uv/). First, install uv:
+
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+Then install MVT as a command-line tool:
+
+```bash
+uv tool install mvt
+```
+
+For alternative installation options and known issues, please refer to the [documentation](https://docs.mvt.re/en/latest/install/) as well as [GitHub Issues](https://github.com/mvt-project/mvt/issues).
+
 
 ## Usage
 
-MVT provides two commands `mvt-ios` and `mvt-android` with the following subcommands available:
+MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
 
-* `mvt-ios`:
-    * `check-backup`: Extract artifacts from an iTunes backup
-    * `check-fs`: Extract artifacts from a full filesystem dump
-    * `check-iocs`: Compare stored JSON results to provided indicators
-    * `decrypt-backup`:  Decrypt an encrypted iTunes backup
-* `mvt-android`:
-    * `check-backup`: Check an Android Backup
-    * `download-apks`: Download all or non-safelisted installed APKs
+### Shell completion
+
+MVT can generate shell completion scripts for Bash, Zsh, and Fish:
+
+```bash
+mvt-ios completion
+mvt-android completion
+```
+
+The commands print setup instructions by default. To generate a completion script directly, pass the shell name:
+
+```bash
+mvt-ios completion bash
+mvt-android completion zsh
+```
+
+MVT only writes completion files or shell configuration when `--install` is passed. See the [command completion documentation](https://docs.mvt.re/en/latest/command_completion/) for details.
+Module-running `check-*` commands can load custom Python modules with
+`--load-module PATH` or from a folder set in `MVT_CUSTOM_MODULES`. See the
+[development documentation](https://docs.mvt.re/en/latest/development/) for
+details.
+
+Users can also add top-level commands to `mvt-ios` and `mvt-android` from
+installed Python packages or local files and folders. See the
+[custom CLI command documentation](https://docs.mvt.re/en/latest/development/custom_commands/)
+for the plugin entry points and `--load-command` interface.
 
-Check out [the documentation to see how to use them](https://mvt.readthedocs.io/en/latest/).
 
 ## License
 
-The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals. Therefore, the goal of this license is to prohibit the use of MVT (and any other software licensed the same) for the purpose of *adversarial forensics*.
-
-In order to achieve this, MVT is released under an adaptation of [Mozilla Public License v2.0](https://www.mozilla.org/MPL). This modified license includes a new clause 3.0, "Consensual Use Restriction" which permits the use of the licensed software (and any *"Larger Work"* derived from it) exclusively with the explicit consent of the person/s whose data is being extracted and/or analysed (*"Data Owner"*).
-
-[Read the LICENSE](https://github.com/mvt-project/mvt/blob/main/LICENSE)
+The purpose of MVT is to facilitate the ***consensual forensic analysis*** of devices of those who might be targets of sophisticated mobile spyware attacks, especially members of civil society and marginalized communities. We do not want MVT to enable privacy violations of non-consenting individuals.  In order to achieve this, MVT is released under its own license. [Read more here.](https://docs.mvt.re/en/latest/license/)

SECURITY.md

@@ -0,0 +1,62 @@
+# Reporting security issues
+
+Thank you for your interest in reporting security issues and vulnerabilities! Security research is of utmost importance and we take all reports seriously. If you discover an issue please report it to us right away!
+
+Please DO NOT file a public issue, instead send your report privately to the MVT maintainers at Amnesty International via `security [at] amnesty [dot] tech`.
+
+You can also write PGP-encrypted emails to key `CFBF9698DCA8EB2A80F48ADEA035A030FA04ED13`. The corresponding PGP public key is lited below.
+
+```
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGlFPwsBEADQ+d7SeHrFPYv3wPOjWs2oMpp0DPdfIyGbg+iYWOC36FegZhKY
++WeK96GqJWt8wD6kwFUVwQI795WZrjSd1q4a7wR+kj/h7xlRB6ZfVICA6O5DOOm6
+GNMvqy7ESm8g1XZDpb2u1BXmSS9X8f6rjB0e86kYsF1mB5/2USTM63jgDs0GGTkZ
+Q1z4Mq4gYyqH32b3gvXkbb68LeQmONUIM3cgmec9q8/pNc1l7fcoLWhOVADRj17Q
+plisa/EUf/SYqdtk9w7EHGggNenKNwVM235mkPcMqmE72bTpjT6XCxvZY3ByG5yi
+7L+tHJU45ZuXtt62EvX03azxThVfSmH/WbRk8lH8+CW8XMmiWZphG4ydPWqgVKCB
+2UOXm+6CQnKA+7Dt1AeK2t5ciATrv9LvwgSxk5WKc3288XFLA6eGMrTdQygYlLjJ
++42RSdK/7fCt/qk4q13oUw8ZTVcCia98uZFi704XuuYTH6NrntIB7j/0oucIS4Y9
+cTWNO5LBerez4v8VI4YHcYESPeIWGFkXhvJzo0VMg1zidBLtiPoGF2JKZGwaK7/p
+yY1xALskLp4H+5OY4eB1kf8kl4vGsEK8xA/NNzOiapVmwBXpvVvmXIQJE2k+olNf
+sAuyB8+aO1Ws7tFYt3D+olC7iaprOdK7uA4GCgmYYhq6QQPg+cxfczgHfwARAQAB
+tD1TZWN1cml0eSBMYWIgYXQgQW1uZXN0eSBJbnRlcm5hdGlvbmFsIDxzZWN1cml0
+eUBhbW5lc3R5LnRlY2g+iQJRBBMBCAA7FiEEz7+WmNyo6yqA9IreoDWgMPoE7RMF
+AmlFPwsCGwMFCwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQoDWgMPoE7RNr
+2w//a88uP90uSN6lgeIwKsHr1ri27QIBbzCV6hLN/gZBFR2uaiOn/xfFDbnR0Cjo
+5nMCJCT1k4nrPbMTlfmWLCD+YKELBzVqWlw4J2SOg3nznPl2JrL8QBKjwts0sF+h
+QbRWDsT54wBZnl6ZJJ79eLShNTokBbKnQ7071dMrENr5e2P2sClQXyiIc51ga4FM
+fHyhsx+GsrdiZNd2AH8912ljW1GuEi3epTO7KMZprmr37mjpZSUToiV59Yhl1Gbo
+2pixkYJqi62DG02/gTpCjq9NH3cEMxcxjh4E7yCA8ggLG6+IN6woIvPIdOsnQ+Yj
+d3H4rMNBjPSKoL+bdHILkCnp5HokcbVjNY3QAyOAF4qWhk4GtgpTshwxUmb4Tbay
+tWLJC2bzjuUBxLkGzMVFfU3B96sVS4Fi0sBaEMBtHskl2f45X8LJhSq//Lw/2L/8
+34uP/RxDSn+DPvj/yqMpekdCcmeFSTX1A19xkPcc0rVhMRde4VL338R86vzh0gMI
+1LySDAhXZyVWzrQ5s3n6N3EvCaHCn3qu7ieyFJifCSR7gZqevCEznMQRVpkMTzUt
+rk13Z6NOOb4IlTW7HFoY3omJG8Z5jV4kMIE7n6nb0qpNYQiG+YvjenQ3VrMoISyh
+lpS2De8+oOtwrxBVX3+qKWvQqzufeE3416kw2Z+5mxH7bx25Ag0EaUU/CwEQALyZ
+b+kwLN1yHObTm2yDBEn5HbCT3H1GremvPNmbAaTnfrjUngoKa8MuWWzbX5ptgmZR
+UpYY/ylOYcgGydz58vUNrPlhIZT9UhmiifPgZLEXyd0uFpr/NsbRajHMkK10iEZf
+h5bHNobiB7pGCu4Uj9e1cMiIZ4yEaYeyXYUoNHf6ISP39mJhHy6ov5yIpm9q0wzm
+tGUQPupxGXmEZlOPr3lxqXQ3Ekdv6cWDY5r/oOq71QJ/HUQ13QUuGFIbhnMbT8zd
+zaS6f/v772YKsWPc4NNUhtlf25VnQ4FuUtjCe3p6iYP4OVD8gJm0GvXyvyTuiQbL
+CSk/378JiNT7nZzYXxrWchMwvEoMIU55+/UaBc50HI5xvDQ858CX7PYGiimcdsO1
+EkQzhVxRfjlILfWrC2lgt+H5qhTn4Fah250Xe1PnLjXGHVUQnY/f3MFeiWQgf92b
+02+MfvOeC5OKttP1z5lcx6RFWCIa1E/u8Nj7YrH9hk0ZBRAnBaeAncDFY8dfX2zX
+VMoc0dV16gM7RrZ6i7D3CG3eLLkQlX0jbW9dzTuG/3f098EWB1p8vOfS/RbNCBRX
+jqGiqacL/aFF3Ci3nQ4O5tSv1XipbgrUhvXnwm9pxrLPS/45iaO59WN4RRGWLLQ7
+LHmeBxoa9avv0SdBYUL+eBxY46GXb/j5VLzHYhSnABEBAAGJAjYEGAEIACAWIQTP
+v5aY3KjrKoD0it6gNaAw+gTtEwUCaUU/CwIbDAAKCRCgNaAw+gTtEyvsEACnyFFD
+alOZTrrJTXNnUejuiExLh+qTO3T91p5bte597jpwCZnYGwkxEfffsqqhlY6ftEOf
+d5tNWE5isai4v8XCbplWomz4KBpepxcn2b+9o5dSyr1vohEFuCJziZDsta1J2DX5
+IE9U48kTgLDfdIBhuOyHNRkvXRHP2OVLCaiw4d9q+hlrraR8pehHt2BJSxh+QZoe
+n0iHvIZCBIUA45zLEGmXFpNTGeEf2dKPp3xOkAXOhAMPptE0V1itkF3R7kEW4aFO
+SZo8L3C1aWSz/gQ4/vvW5t1IJxirNMUgTMQFvqEkAwX3fm6GCxlgRSvTTRXdcrS8
+6qyFdH1nkCNsavPahN3N2RGGIlWtODEMTO1Hjy0kZtTYdW+JH9sendliCoJES+yN
+DjM125SgdAgrqlSYm/g8n9knWpxZv1QM6jU/sVz1J+l6/ixugL2i+CAL2d6uv4tT
+QmXnu7Ei4/2kHBUu3Lf59MNgmLHm6F7AhOWErszSeoJKsp+3yA1oTT/npz67sRzY
+VVyxz4NBIollna59a1lz0RhlWzNKqNB27jhylyM4ltdzHB7r4VMAVJyttozmIIOC
+35ucYxl5BHLuapaRSaYHdUId1LOccYyaOOFF/PSyCu9dKzXk7zEz2HNcIboWSkAE
+8ZDExMYM4WVpVCOj+frdsaBvzItHacRWuijtkw==
+=JAXX
+-----END PGP PUBLIC KEY BLOCK-----
+```

dev/mvt-android

@@ -1,13 +0,0 @@
-#!/usr/bin/env python3
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from mvt import android
-android.cli()

dev/mvt-ios

@@ -1,13 +0,0 @@
-#!/usr/bin/env python3
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from mvt import ios
-ios.cli()

docs/android/adb.md

@@ -0,0 +1,29 @@
+# ADB Analysis Removed
+
+The ability to analyze Android devices directly over the [Android Debug Bridge (adb)](https://developer.android.com/studio/command-line/adb) has been removed from MVT.
+
+Use [AndroidQF](https://github.com/mvt-project/androidqf) to collect forensic artifacts from an Android device, then analyze the collected output with:
+
+```bash
+mvt-android check-androidqf /path/to/androidqf-output
+```
+
+For a standalone Android bug report, use:
+
+```bash
+mvt-android check-bugreport /path/to/bugreport.zip
+```
+
+## Reasons for Removal
+
+1. **Inconsistent Data Collection Across Devices**
+   Android devices vary significantly in system architecture, security policy, and diagnostic log availability. This made it difficult to collect reliable forensic data across devices.
+
+2. **Incomplete Forensic Data Acquisition**
+   Direct ADB analysis did not retrieve a complete forensic snapshot. Critical artifacts such as full bug reports could be missing.
+
+3. **Duplicated Analysis Paths**
+   Similar artifacts were parsed through separate ADB, AndroidQF, and bugreport modules, which made behavior harder to keep consistent.
+
+4. **Workflow Consistency**
+   MVT now focuses on analyzing acquired artifacts rather than interacting with live devices directly.

docs/android/backup.md

@@ -1,38 +1,65 @@
-# Checking SMSs from Android backup
+# Check an Android Backup (SMS messages)
 
-Some attacks against Android phones are done by sending malicious links by SMS. The Android backup feature does not allow to gather much information that can be interesting for a forensic analysis, but it can be used to extract SMSs and check them with MVT.
+Android supports generating a backup archive of all the installed applications which supports it. However, over the years this functionality has been increasingly abandoned in favor of enabling users to remotely backup their personal data over the cloud. App developers can therefore decide to opt out from allowing the apps' data from being exported locally.
 
-To do so, you need to connect your Android device to your computer. You will then need to [enable USB debugging](https://developer.android.com/studio/debug/dev-options#enable>) on the Android device.
+At the time of writing, the Android Debug Bridge (adb) command to generate backups is still available but marked as deprecated.
 
-If this is the first time you connect to this device, you will need to approve the authentication keys through a prompt that will appear on your Android device.
+That said, most versions of Android should still allow to locally backup SMS messages, and since messages are still a prime vehicle for phishing and malware attacks, you might still want to take advantage of this functionality while it is supported.
 
-Then you can use adb to extract the backup for SMS only with the following command:
+## Generate a backup
+
+Because `mvt-android check-backup` currently only supports checking SMS messages, you can indicate to backup only those:
 
 ```bash
-adb backup com.android.providers.telephony
+adb backup -nocompress com.android.providers.telephony
 ```
 
-You will need to approve the backup on the phone and potentially enter a password to encrypt the backup. The backup will then be stored in a file named `backup.ab`.
+In case you nonetheless wish to take a full backup, you can do so with
 
-You will need to use [Android Backup Extractor](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
 ```bash
-java -jar ~/Download/abe.jar unpack backup.ab backup.tar
+adb backup -nocompress -all
+```
+
+Some recent phones will enforce the utilisation of a password to encrypt the backup archive. In that case, the password will obviously be needed to extract and analyse the data later on.
+
+## Unpack and check the backup
+
+MVT includes a partial implementation of the Android Backup parsing, because of the implementation difference in the compression algorithm between Java and Python. The `-nocompress` option passed to adb in the section above allows to avoid this issue. You can analyse and extract SMSs from the backup directly with MVT:
+
+```bash
+$ mvt-android check-backup --output /path/to/results/ /path/to/backup.ab
+14:09:45 INFO     [mvt.android.cli] Checking ADB backup located at: backup.ab
+         INFO     [mvt.android.modules.backup.sms] Running module SMS...
+         INFO     [mvt.android.modules.backup.sms] Processing SMS backup file at
+                  apps/com.android.providers.telephony/d_f/000000_sms_backup
+         INFO     [mvt.android.modules.backup.sms] Extracted a total of 64 SMS messages
+```
+
+If the backup is encrypted, MVT will prompt you to enter the password. A backup password can also be provided with the `--backup-password` command line option or through the `MVT_ANDROID_BACKUP_PASSWORD` environment variable. The same options can also be used to when analysing an encrypted backup collected through AndroidQF in the `mvt-android check-androidqf` command:
+
+```bash
+$ mvt-android check-backup --backup-password "password123" --output /path/to/results/ /path/to/backup.ab
+```
+
+Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by MVT. Any matches will be highlighted in the terminal output.
+
+## Alternative ways to unpack and check the backup
+
+If you encounter an issue during the analysis of the backup, you can alternatively use [Android Backup Extractor (ABE)](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
+
+```bash
+java -jar ~/path/to/abe.jar unpack backup.ab backup.tar
 tar xvf backup.tar
 ```
 
-(If the backup is encrypted, the password will be asked by Android Backup Extractor).
+If the backup is encrypted, ABE will prompt you to enter the password.
 
-You can then extract SMSs containing links with MVT:
+Alternatively, [ab-decrypt](https://github.com/joernheissler/ab-decrypt) can be used for that purpose.
 
-```bash
-$ mvt-android check-backup --output sms .
-16:18:38 INFO     [mvt.android.cli] Checking ADB backup located at: .
-         INFO     [mvt.android.modules.backup.sms] Running module SMS...
-         INFO     [mvt.android.modules.backup.sms] Processing SMS backup
-                  file at ./apps/com.android.providers.telephony/d_f/000
-                  000_sms_backup
-16:18:39 INFO     [mvt.android.modules.backup.sms] Extracted a total of
-                  64 SMS messages containing links
-```
+You can then extract SMSs with MVT by passing the folder path as parameter instead of the `.ab` file: `mvt-android check-backup --output /path/to/results/ /path/to/backup/` (the path to backup given should be the folder containing the `apps` folder).
 
-Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by mvt. Any matches will be highlighted in the terminal output.
+When an output folder is specified, URLs extracted from SMS and MMS messages
+are also written to `urls.json`. Each entry contains the URL, its expanded
+destination when MVT resolved a shortened URL during indicator checking, the
+message timestamp, and the `sms` source. The same file is created by
+`check-androidqf` when its nested Android backup contains messages with URLs.

docs/android/download_apks.md

@@ -1,24 +0,0 @@
-# Downloading APKs from an Android phone
-
-In order to use `mvt-android` you need to connect your Android device to your computer. You will then need to [enable USB debugging](https://developer.android.com/studio/debug/dev-options#enable>) on the Android device.
-
-If this is the first time you connect to this device, you will need to approve the authentication keys through a prompt that will appear on your Android device.
-
-Now you can launch `mvt-android` and specify the `download-apks` command and the path to the folder where you want to store the extracted data:
-
-```bash
-mvt-android download-apks --output /path/to/folder
-```
-
-Optionally, you can decide to enable lookups of the SHA256 hash of all the extracted APKs on [VirusTotal](https://www.virustotal.com) and/or [Koodous](https://www.koodous.com). While these lookups do not provide any conclusive assessment on all of the extracted APKs, they might highlight any known malicious ones:
-
-```bash
-mvt-android download-apks --output /path/to/folder --virustotal
-mvt-android download-apks --output /path/to/folder --koodous
-```
-
-Or, to launch all available lookups::
-
-```bash
-mvt-android download-apks --output /path/to/folder --all-checks
-```

docs/android/intrusion_logs.md

@@ -0,0 +1,82 @@
+# Check Android Intrusion Logs
+
+Recent versions of Android can produce structured *Intrusion Logs* — newline-delimited JSON records derived from the platform's [SecurityLog API](https://developer.android.com/reference/android/app/admin/SecurityLog). Intrusion Logging is offered as a new option under Android's **Advanced Protection Mode**, which users can opt into on their device; no MDM or device-policy configuration is required. When enabled, these logs provide a high-fidelity record of process starts, DNS queries, outbound network connections, ADB activity, keyguard events, and other security-relevant operations. The initial Intrusion Logging feature was released for Android 16 in May 2026. The feature and supported events is likely to be expanded over time.
+
+For background on how this data source was introduced and why it is forensically valuable, see the Amnesty International Security Lab announcement: [Android Intrusion Logging as a new source of data for consensual forensic analysis](https://securitylab.amnesty.org/latest/2026/05/android-intrusion-logging-as-a-new-source-of-data-for-consensual-forensic-analysis/).
+
+## Recommended workflow: collect with AndroidQF
+
+[AndroidQF](https://github.com/mvt-project/androidqf) is the recommended way to acquire data from an Android device for analysis with MVT. During acquisition AndroidQF will prompt the user to also collect intrusion logs from the device, and writes them into an `intrusion-logs/` subdirectory of the acquisition output.
+
+When you analyse such an acquisition with `mvt-android check-androidqf`, MVT automatically detects the `intrusion-logs/` directory and runs the same intrusion-log checks described below — there is no need to invoke a separate command:
+
+```bash
+mvt-android check-androidqf --output /path/to/results/ /path/to/androidqf-output/
+```
+
+The device timezone is read from the AndroidQF acquisition (`getprop.txt`) and applied to event timestamps automatically.
+
+## Standalone command: `check-intrusion-logs`
+
+The `mvt-android check-intrusion-logs` command runs the intrusion-log analysis directly against a set of log files. Prefer the AndroidQF workflow above; use the standalone command when the intrusion logs were collected outside of an AndroidQF acquisition, or when re-analysing only a set of intrusion logs.
+
+## Expected input
+
+`check-intrusion-logs` accepts either:
+
+- a **directory** containing one or more `.txt` files (recursively), or
+- a **`.zip` archive** containing such `.txt` files (nested `.zip` archives are also walked).
+
+Each `.txt` file is expected to contain newline-delimited JSON, with one JSON object per line. Each object wraps a single event under a top-level key indicating its type, for example:
+
+```json
+{"dns_event": {"event_time": 1746979200000, "hostname": "example.com", "ip_addresses": ["93.184.216.34"], "package_name": "com.example.app"}}
+{"connect_event": {"event_time": 1746979201000, "ip_address": "93.184.216.34", "port": 443, "package_name": "com.example.app"}}
+{"security_event": {"event_time": 1746979202000, "tag": 210005, "data": ["..."]}}
+```
+
+Identical events that appear across multiple overlapping log files (e.g. daily rotations) are de-duplicated on a first-seen basis.
+
+## Running the analysis
+
+```bash
+mvt-android check-intrusion-logs --output /path/to/results/ /path/to/intrusion-logs/
+```
+
+A `.zip` archive can be passed directly in place of the directory:
+
+```bash
+mvt-android check-intrusion-logs --output /path/to/results/ /path/to/intrusion-logs.zip
+```
+
+### Options
+
+| Option | Description |
+| --- | --- |
+| `-i, --iocs PATH` | Path to a STIX2 indicator file. May be passed multiple times. |
+| `-o, --output PATH` | Directory where JSON results and the timeline CSV will be written. |
+| `-l, --list-modules` | List the available intrusion-log modules and exit. |
+| `-m, --module NAME` | Run a single module (e.g. `DnsEvent`) instead of all of them. |
+| `-t, --timezone TZ` | IANA timezone name for the device (e.g. `Europe/Paris`). When set, event timestamps are converted to the device's local time instead of UTC. |
+| `-v, --verbose` | Verbose logging. |
+
+## Modules
+
+The command runs the following modules over the parsed events:
+
+- **`DnsEvent`** — DNS resolution events. Hostnames and resolved IP addresses are checked against domain indicators, and the requesting `package_name` is checked against app-identifier indicators.
+- **`ConnectEvent`** — Outbound network connection events. Destination IPs (with localhost addresses skipped) are checked against domain indicators, and `package_name` is checked against app-identifier indicators.
+- **`SecurityEvent`** — Security log events keyed by Android `SecurityLog` tag IDs (e.g. `app_process_start`, `adb_shell_cmd`, `keyguard_dismissed`, `os_startup`, `cert_*` events). These are surfaced in the timeline to help reconstruct device activity around suspected events.
+
+All three modules share a single pre-parsing pass over the input, so adding more modules in the future does not multiply I/O cost. Additional modules will be added in the future to support new event types which are generated by the Intrusion Logging feature.
+
+## Interpreting results
+
+A successful IOC match raises a `CRITICAL` alert that includes the matched indicator, the offending event, and the event timestamp. Alerts are summarised at the end of the run and persisted alongside the per-module JSON results.
+
+When `--timezone` is provided, timestamps in the timeline and JSON output reflect the device's local wall-clock time. Otherwise timestamps are in UTC, consistent with the rest of MVT.
+
+## Limitations
+
+- This page assumes the intrusion logs have already been collected from the device. The recommended collection path is via AndroidQF (see above); intrusion logging itself must have been enabled on the device beforehand by opting into Android's Advanced Protection mode and also enabling the optional Intrusion Logging feature (see the [Amnesty blog post](https://securitylab.amnesty.org/latest/2026/05/android-intrusion-logging-as-a-new-source-of-data-for-consensual-forensic-analysis/) for details).
+- As with all IOC-based analysis, public indicators alone are not sufficient to conclude that a device is uncompromised. See the [Indicators of Compromise](../iocs.md) page for context.

docs/android/methodology.md

@@ -1,8 +1,75 @@
 # Methodology for Android forensic
 
-For different technical reasons, it is more complex to do a forensic analysis of an Android phone.
+Unfortunately Android devices provide fewer complete forensically useful datasources than their iOS cousins. Unlike iOS, the Android backup feature only provides a limited about of relevant data.
 
-Currently MVT allows to perform two different checks on an Android phone:
+Android diagnostic logs such as *bugreport files* can be inconsistent in format and structure across different Android versions and device  vendors. The limited diagnostic information available makes it difficult to triage potential compromises, and because of this `mvt-android` capabilities are limited as well.
 
-* Download APKs installed in order to analyze them
-* Extract Android backup in order to look for suspicious SMS
+However, not all is lost.
+
+## Check Android devices with AndroidQF and MVT
+
+The [AndroidQF](https://github.com/mvt-project/androidqf) tool can be used to collect a wide range of forensic artifacts from an Android device including an Android backup, a bugreport file, and a range of system logs. MVT natively supports analyzing the generated AndroidQF output for signs of device compromise.
+
+### Why Use AndroidQF?
+
+- **Complete and raw data extraction**
+  AndroidQF collects full forensic artifacts using an on-device forensic collection agent, ensuring that no crucial data is overlooked. The data collection does not depended on the shell environment or utilities available on the device.
+
+- **Consistent and standardized output**
+  By collecting a predefined and complete set of forensic files, AndroidQF ensures consistency in data acquisition across different Android devices.
+
+- **Future-proof analysis**
+  Since the full forensic artifacts are preserved, analysts can extract new evidence or apply updated analysis techniques without requiring access to the original device.
+
+- **Cross-platform tool without dependencies**
+  AndroidQF is a standalone Go binary which can be used to remotely collect data from an Android device without the device owner needing to install MVT or a Python environment.
+
+### Workflow for Android Forensic Analysis with AndroidQF
+
+With AndroidQF the analysis process is split into a separate data collection and data analysis stages.
+
+1. **Extract Data Using AndroidQF**
+  Deploy the AndroidQF forensic collector to acquire all relevant forensic artifacts from the Android device.
+
+2. **Analyze Extracted Data with MVT**
+  Use the `mvt-android check-androidqf` command to perform forensic analysis on the extracted artifacts.
+
+By separating artifact collection from forensic analysis, this approach ensures a more reliable and scalable methodology for Android forensic investigations.
+
+For more information, refer to the [AndroidQF project documentation](https://github.com/mvt-project/androidqf).
+
+### VirusTotal package lookups
+
+AndroidQF records APK file hashes in `packages.json`. MVT can optionally look up non-system APK hashes on VirusTotal while checking an AndroidQF acquisition:
+
+```bash
+MVT_VT_API_KEY=<key> mvt-android check-androidqf --virustotal /path/to/androidqf-output
+```
+
+The `--virustotal` option is disabled by default because it sends APK hashes to VirusTotal and requires network access. It uses the `VT_API_KEY` MVT configuration value, which can also be provided through the `MVT_VT_API_KEY` environment variable.
+
+To avoid exhausting free VirusTotal API quotas, MVT waits 16 seconds between package hash requests by default. Use `--delay` to change the delay, or `--delay 0` to disable throttling:
+
+```bash
+mvt-android check-androidqf --virustotal --delay 30 /path/to/androidqf-output
+```
+
+## Android Intrusion Logs
+
+On devices where the user has opted into Android's [**Advanced Protection Mode**](https://support.google.com/android/answer/16339980) and turned on the optional Intrusion Logging featrue, Android can create and archive structured *Intrusion Logs* in an encrypted format. These logs record DNS queries, outbound network connections, process starts, ADB activity and other security-relevant events, and are a high-fidelity complement to the rest of an AndroidQF acquisition. The logs are generated on-device and encrypted before being stored in the Google account associated with the device. The encryption key is protected by the user device PIN. The intrusion log data is not accessible to Google.
+
+AndroidQF will prompt the user to download, decrypt and collect device intrusion logs as part of an acquisition. When they are present, `mvt-android check-androidqf` will automatically run the intrusion-log checks alongside the other AndroidQF modules — no extra command is required. This is the recommended workflow for Android forensic analysis with MVT.
+
+For cases where intrusion logs were collected outside of an AndroidQF acquisition, the standalone `mvt-android check-intrusion-logs` command can analyse them directly. See [Check Android Intrusion Logs](intrusion_logs.md) for details, and the [feature announcment from Amnesty International's Security Lab](https://securitylab.amnesty.org/latest/2026/05/android-intrusion-logging-as-a-new-source-of-data-for-consensual-forensic-analysis/) for background on the data source.
+
+## Android Debug Bridge analysis removed
+
+The ability to analyze Android devices directly over ADB has been removed from MVT. Direct extraction of data from ADB was error-prone and frequently resulted in inconsistent data collection between ADB and AndroidQF acquisitions. Use AndroidQF for device acquisition and `mvt-android check-androidqf` for analysis.
+
+## Check an Android Backup (SMS messages)
+
+Although Android backups are becoming deprecated, it is still possible to generate one. Unfortunately, because apps these days typically favor backup over the cloud, the amount of data available is limited.
+
+The `mvt-android check-androidqf` command will automatically check an Android backup and SMS messages if an SMS backup is included in the AndroidQF extraction.
+
+The  `mvt-android check-backup` command can also be used directly with an Android backup file.

docs/command_completion.md

@@ -0,0 +1,66 @@
+# Command Completion
+
+MVT utilizes the [Click](https://click.palletsprojects.com/en/stable/) library for creating its command line interface.
+
+Click provides tab completion support for Bash (version 4.4 and up), Zsh, and Fish.
+
+To enable it, you need to register a completion script with your shell, which varies depending on the shell you are using.
+
+The following describes how to generate the command completion scripts and add them to your shell configuration.
+
+> **Note: You will need to start a new shell for the changes to take effect.**
+
+### For Bash
+
+```bash
+# Generate bash completion scripts
+mvt-ios completion bash > ~/.mvt-ios-complete.bash
+mvt-android completion bash > ~/.mvt-android-complete.bash
+```
+
+Add the following to `~/.bashrc`:
+```bash
+# source mvt completion scripts
+[ -f ~/.mvt-ios-complete.bash ] && . ~/.mvt-ios-complete.bash
+[ -f ~/.mvt-android-complete.bash ] && . ~/.mvt-android-complete.bash
+```
+
+### For Zsh
+
+```bash
+# Generate zsh completion scripts
+mvt-ios completion zsh > ~/.mvt-ios-complete.zsh
+mvt-android completion zsh > ~/.mvt-android-complete.zsh
+```
+
+Add the following to `~/.zshrc`:
+```bash
+# source mvt completion scripts
+[ -f ~/.mvt-ios-complete.zsh ] && . ~/.mvt-ios-complete.zsh
+[ -f ~/.mvt-android-complete.zsh ] && . ~/.mvt-android-complete.zsh
+```
+
+### For Fish
+
+```bash
+# Generate fish completion scripts
+mkdir -p ~/.config/fish/completions
+mvt-ios completion fish > ~/.config/fish/completions/mvt-ios.fish
+mvt-android completion fish > ~/.config/fish/completions/mvt-android.fish
+```
+
+Fish loads completion files from `~/.config/fish/completions` automatically.
+
+### Automatic Installation
+
+MVT can write the completion file and update the relevant shell configuration for Bash and Zsh when you pass `--install`:
+
+```bash
+mvt-ios completion bash --install
+mvt-android completion bash --install
+```
+
+Replace `bash` with `zsh` or `fish` as needed. For Fish, `--install` writes the completion file into `~/.config/fish/completions`.
+
+For more information, visit the official [Click Docs](https://click.palletsprojects.com/en/stable/shell-completion/#enabling-completion).
+

docs/development/custom_commands.md

@@ -0,0 +1,111 @@
+# Custom CLI Commands
+
+MVT can load additional top-level commands into `mvt-ios` and `mvt-android`.
+Custom commands are different from [custom forensic modules](index.md#custom-modules):
+commands add new CLI operations, while modules add analysis steps to existing
+`check-*` commands.
+
+!!! warning
+
+    Custom commands run as trusted Python code inside the MVT process. Install
+    or load commands only from sources you trust. MVT does not sandbox
+    third-party commands, and the MVT maintainers do not maintain them.
+
+## Install a Command Package
+
+Python packages can register a Click command or group for either MVT CLI. A
+minimal package can expose this command from `my_mvt_plugin.py`:
+
+```python
+import click
+
+
+@click.command()
+@click.argument("path", type=click.Path(exists=True))
+def summarize(path):
+    """Summarize an acquisition."""
+    click.echo(f"Summarizing {path}")
+```
+
+Register the object in the package's `pyproject.toml`. The entry-point name is
+the command users invoke:
+
+```toml
+[project.entry-points."mvt.ios.cli_plugins"]
+summarize = "my_mvt_plugin:summarize"
+
+[project.entry-points."mvt.android.cli_plugins"]
+summarize = "my_mvt_plugin:summarize"
+```
+
+Use only the iOS or Android group if the command is platform-specific. After
+installing the package in the same environment as MVT, it appears directly in
+the appropriate CLI:
+
+```bash
+mvt-ios summarize ./ios-backup
+mvt-android summarize ./androidqf-output
+```
+
+For a `pipx` installation of MVT, inject the plugin into MVT's environment:
+
+```bash
+pipx inject mvt my-mvt-plugin
+```
+
+When MVT is installed in an active virtual environment, install the plugin with
+`pip` in that environment.
+
+Command packages that need their own settings, such as an API key, should store
+them in a namespaced [plugin configuration file](plugin_configuration.md)
+rather than in MVT's own `config.yaml`.
+
+## Load a Command File
+
+For local commands that are not packaged, create a Python file that exports one
+Click command or group named `cli`:
+
+```python
+import click
+
+
+@click.command("case-summary")
+@click.argument("path", type=click.Path(exists=True))
+def cli(path):
+    """Summarize a case directory."""
+    click.echo(f"Summarizing {path}")
+```
+
+Pass the file before the custom command name:
+
+```bash
+mvt-ios --load-command ./case_summary.py case-summary ./ios-backup
+```
+
+`--load-command` can be repeated and also accepts a folder. MVT loads
+non-hidden top-level `*.py` files in sorted order and skips `__init__.py`.
+Every loaded file must export one `cli` object.
+
+To load a file or folder on every invocation, set the platform-specific
+environment variable:
+
+```bash
+export MVT_IOS_CUSTOM_COMMANDS=./ios_commands
+export MVT_ANDROID_CUSTOM_COMMANDS=./android_commands
+```
+
+## Naming and Errors
+
+Built-in MVT commands cannot be replaced. External command names must also be
+unique; when installed packages or environment paths collide, MVT keeps the
+first command and logs a warning. A collision from an explicit
+`--load-command` is a usage error.
+
+A package entry point or environment command that cannot be imported appears
+as a marked broken command without preventing other MVT commands from working.
+Invoke that command to see its package or file source and the underlying error.
+An invalid command supplied explicitly with `--load-command` fails immediately
+with a usage error.
+
+Installed command packages use the entry-point name as the CLI command name.
+The entry point must resolve to a `click.Command` or `click.Group`.

docs/development/index.md

@@ -0,0 +1,229 @@
+# Development
+
+The Mobile Verification Toolkit team welcomes contributions of new forensic modules or other contributions which help improve the software.
+
+## Local environment
+
+MVT uses `uv` for dependency management. To install the project and development dependencies from the locked environment, run:
+
+```bash
+make install
+```
+
+## Testing
+
+MVT uses `pytest` for unit and integration tests. Code style consistency is maintained with `ruff` and `mypy`. All can
+be run automatically with:
+
+```bash
+make check
+```
+
+Run these tests before making new commits or opening pull requests.
+
+## Module dependencies
+
+Modules can require other modules to run first by declaring their classes in
+`dependencies`. The command runner uses a stable topological ordering, so the
+existing module list order is preserved wherever dependency constraints allow.
+
+```python
+class DependentModule(MVTModule):
+    dependencies = (PrerequisiteModule,)
+
+    def run(self):
+        prerequisite_results = self.get_dependency_results(PrerequisiteModule)
+```
+
+Selecting a single module also runs its transitive dependencies.
+
+A module can only depend on modules the command it runs in also has. When a
+declared dependency is not among them, the command logs a warning naming the
+module and the missing dependency, skips that module and everything depending
+on it, and runs the rest of the analysis. Selecting such a module with
+`--module` therefore leaves nothing to run, which the warning explains.
+
+A cycle in the dependency graph is a programming error rather than a
+configuration problem: the command logs a warning and runs no modules at all.
+
+## Custom modules
+
+Module-running `check-*` commands can load custom modules from Python files that
+are not installed as part of MVT. Load one file with:
+
+```bash
+mvt-ios check-backup --load-module ./example_module.py --output ./out ./backup
+```
+
+You can also load a folder. MVT loads non-hidden top-level `*.py` files in
+sorted order and skips `__init__.py`:
+
+```bash
+mvt-ios check-fs --load-module ./custom_modules ./filesystem-dump
+```
+
+Set `MVT_CUSTOM_MODULES` to load a folder for every module-running command. This
+folder is loaded before any `--load-module` path:
+
+```bash
+MVT_CUSTOM_MODULES=./custom_modules mvt-android check-bugreport ./bugreport.zip
+```
+
+Custom modules are normal `MVTModule` subclasses:
+
+```python
+from mvt.common.module import MVTModule
+
+
+class ExampleCustomModule(MVTModule):
+    supported_commands = (("ios", "check-backup"), ("ios", "check-fs"))
+    slug = "example_custom_module"
+
+    def run(self):
+        self.results = [{"message": "custom module ran"}]
+
+    def check_indicators(self):
+        pass
+
+    def serialize(self, result):
+        return None
+```
+
+Use `supported_commands` to declare the platform/command pairs a module
+supports. Empty `supported_commands` means the module will not run and MVT logs
+a warning. This explicit declaration is required for every command. Supported
+pairs are:
+
+```python
+("ios", "check-backup")
+("ios", "check-fs")
+("ios", "check-iocs")
+("ios", "check-sysdiagnose")
+("android", "check-backup")
+("android", "check-bugreport")
+("android", "check-androidqf")
+("android", "check-intrusion-logs")
+("android", "check-iocs")
+```
+
+Custom modules can depend on existing MVT module classes. Dependencies are
+resolved with the same ordering logic as built-in modules, and custom modules
+are appended after built-ins before ordering:
+
+```python
+from mvt.common.module import MVTModule
+from mvt.ios.modules.backup.manifest import Manifest
+
+
+class DependentCustomModule(MVTModule):
+    supported_commands = (("ios", "check-backup"),)
+    dependencies = (Manifest,)
+
+    def run(self):
+        manifest_results = self.get_dependency_results(Manifest)
+        self.results = [{"manifest_entries": len(manifest_results)}]
+```
+
+## Installed module packages
+
+Python packages can register modules so they load automatically in every
+module-running `check-*` command, without `--load-module` or
+`MVT_CUSTOM_MODULES`. Register an entry point in the `mvt.modules` group in
+the package's `pyproject.toml`:
+
+```toml
+[project.entry-points."mvt.modules"]
+mvt-plugin-amnesty-custom = "mvt_plugin_amnesty_custom:get_modules"
+```
+
+The entry point must resolve to an iterable of `MVTModule` subclasses, or to
+a callable returning one:
+
+```python
+from mvt.common.module import MVTModule
+
+
+class PackagedModule(MVTModule):
+    supported_commands = (("ios", "check-backup"),)
+
+    def run(self):
+        self.results = [{"message": "packaged module ran"}]
+
+
+def get_modules() -> list[type[MVTModule]]:
+    return [PackagedModule]
+```
+
+Installed modules follow the same rules as other custom modules: each module
+must declare `supported_commands`, and dependencies are resolved with the
+standard ordering logic. A broken entry point is skipped with a warning and
+does not prevent MVT from running. As with custom commands, installed module
+packages run as trusted code inside the MVT process, so install only packages
+from sources you trust.
+
+For a `pipx` installation of MVT, inject the package into MVT's environment:
+
+```bash
+pipx inject mvt mvt-plugin-amnesty-custom
+```
+
+Module packages that need their own settings, such as an API key, should store
+them in a namespaced [plugin configuration file](plugin_configuration.md)
+rather than in MVT's own `config.yaml`.
+
+### Naming module packages
+
+Name module packages `mvt-plugin-<name>` (import package `mvt_plugin_<name>`),
+and include the name of the publishing organization or author so packages from
+different groups do not collide: for example, Amnesty International's custom
+modules would be distributed as `mvt-plugin-amnesty-custom` with the import
+package `mvt_plugin_amnesty_custom`.
+
+The prefix makes module packages easy to find on PyPI and keeps their import
+names from clashing with unrelated Python packages. It is a convention, not a
+technical requirement: modules load through the `mvt.modules` entry point
+regardless of what the package is called, and existing packages with other
+names keep working. Note that the prefix is also not a mark of authenticity —
+anyone can publish a package with any available name, so vet a module package
+and its publisher before installing it, whatever it is called.
+
+### Module logging
+
+Modules log through `self.log`, and MVT names the logger for where the module
+came from. MVT's own modules log under their dotted path (for example
+`mvt.ios.modules.mixed.whatsapp`). Everything external is namespaced under
+`mvt.ext` to keep it visually distinct from built-in modules and isolated from
+MVT's internal logger tree:
+
+- Installed packages log under `mvt.ext.<package>`, with the `mvt_plugin_`
+  prefix stripped: modules in `mvt_plugin_amnesty_custom` log as
+  `mvt.ext.amnesty_custom.*`.
+- Files loaded with `--load-module` or `MVT_CUSTOM_MODULES` log as
+  `mvt.ext.<file name>`.
+
+## Auditing loaded modules
+
+Because installed module packages load automatically, MVT records where every
+module came from:
+
+- `--list-modules` groups the available modules by source: MVT itself
+  (with its version), each installed package (with its version and, when
+  installed directly from a repository, the commit), and each file loaded
+  with `--load-module` or `MVT_CUSTOM_MODULES` (with the SHA-256 hash of the
+  file).
+- When a command runs with an `--output` folder, the `command.log` file
+  records one line per module source with the source's version or hash and
+  the list of modules loaded from it.
+
+## Profiling
+
+Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be
+take to avoid inefficient code paths as we add new modules.
+
+MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable.
+
+```bash
+MVT_PROFILE=1 dev/mvt-ios check-backup test_backup
+```
+
+Open an issue or PR if you are encountering significant performance issues when analyzing a device with MVT.

docs/development/plugin_configuration.md

@@ -0,0 +1,149 @@
+# Plugin Configuration
+
+Plugin packages that add [custom CLI commands](custom_commands.md) or
+[modules](index.md#installed-module-packages) often need to store their
+own settings, such as an API key, a server URL or the timestamp of the last
+synchronization. MVT provides a namespaced settings base class so each plugin
+keeps its configuration in its own file, and a data folder for anything else a
+plugin needs to keep on disk.
+
+!!! warning
+
+    Do not write plugin settings to MVT's own `config.yaml`. MVT rewrites that
+    file with the settings it knows about every time it starts, so any other
+    section is deleted.
+
+## Where Settings Are Stored
+
+Each plugin gets one YAML file in a `plugins` folder next to MVT's own
+configuration:
+
+```
+~/.config/mvt/plugins/<plugin name>.yaml
+```
+
+The exact parent folder follows the platform convention used for MVT's
+`config.yaml` (for example `~/Library/Application Support/mvt` on macOS). Use
+`mvt.common.plugin_config.plugin_config_path()` instead of building the path by
+hand.
+
+Plugin names must be lowercase and may only contain letters, digits and dashes,
+matching the `mvt-plugin-<name>` package naming convention. MVT creates the
+`plugins` folder with `0700` permissions and writes the settings files with
+`0600` permissions, because they commonly hold credentials. Files are written
+through a temporary file and moved into place, so an interrupted save never
+leaves a partially written settings file behind.
+
+## Plugin Data Folder
+
+Everything else a plugin keeps on disk, such as a cache, a downloaded artifact
+or synchronization state, belongs in the folder returned by the `data_folder()`
+class method of the plugin's settings class, or by
+`mvt.common.plugin_config.plugin_data_folder()` called with the plugin name if
+the plugin has no settings class:
+
+```
+~/.local/share/mvt/plugin-data/<plugin name>/                # Linux
+~/Library/Application Support/mvt/plugin-data/<plugin name>/ # macOS
+```
+
+The folder sits beside MVT's own data, such as the downloaded indicators. It is
+created if it is missing, with `0700` permissions. Asking for it again returns
+the same path and leaves the contents alone, so a plugin can ask for it every
+time it needs the folder. `ExamplePluginSettings` below is the settings class
+defined in the next section:
+
+```python
+import os
+
+
+def cache_path() -> str:
+    folder = ExamplePluginSettings.data_folder()
+    return os.path.join(folder, "virustotal_lookups_cache.json")
+```
+
+A plugin which has no settings class calls
+`plugin_data_folder("example-plugin")` instead.
+
+Do not fall back on a path of your own such as `~/.cache/example-plugin`: it
+is a Linux-only convention, and MVT will not create it for you.
+
+## Defining Plugin Settings
+
+Subclass `MVTPluginSettings`, set `plugin_name` and declare typed fields with
+defaults:
+
+```python
+from typing import Optional
+
+from mvt.common.plugin_config import MVTPluginSettings
+
+
+class ExamplePluginSettings(MVTPluginSettings):
+    plugin_name = "example-plugin"
+
+    API_KEY: Optional[str] = None
+    MAX_RESULTS: int = 25
+    LAST_SYNC: Optional[str] = None
+```
+
+`load()` returns the current settings and `save()` writes them back:
+
+```python
+from datetime import datetime, timezone
+
+import click
+
+
+def sync():
+    settings = ExamplePluginSettings.load()
+    if not settings.API_KEY:
+        raise click.ClickException(
+            "No API key configured. Set MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY or "
+            "run 'example-plugin configure'."
+        )
+
+    settings.LAST_SYNC = datetime.now(timezone.utc).isoformat()
+    settings.save()
+```
+
+A missing settings file is not an error: the plugin then runs on the field
+defaults and on whatever the environment provides. `save()` only persists the
+values that differ from the defaults, and it never touches MVT's `config.yaml`.
+A settings file that cannot be parsed, or that does not hold a mapping of
+setting names to values, raises a `PluginConfigLoadError` naming the file.
+
+Every subclass that sets its own `plugin_name` gets its own file and its own
+environment namespace. A subclass that does not redefine `plugin_name` inherits
+it, and therefore shares the file and the environment variables of its parent
+class.
+
+## Environment Variables
+
+Every field can also be set with an environment variable. The prefix is
+`MVT_PLUGIN_`, followed by the plugin name upper-cased with dashes replaced by
+underscores, followed by the field name. For the example above:
+
+```bash
+export MVT_PLUGIN_EXAMPLE_PLUGIN_API_KEY=...
+export MVT_PLUGIN_EXAMPLE_PLUGIN_MAX_RESULTS=50
+```
+
+Settings resolve in this order, from highest to lowest priority:
+
+1. Arguments passed to the settings class directly, such as
+   `ExamplePluginSettings(API_KEY="...")`
+2. Environment variables
+3. The plugin's YAML file
+4. The field defaults declared on the settings class
+
+!!! tip
+
+    On shared or multi-user machines, prefer passing API keys through
+    environment variables rather than saving them to the plugin file. `save()`
+    skips every value that the environment currently supplies, so a credential
+    provided that way is not copied into the settings file when a plugin saves
+    an unrelated setting.
+
+Unknown keys in a plugin's YAML file are ignored, so a settings file written by
+a newer version of a plugin does not break an older one.

docs/docker.md

@@ -0,0 +1,48 @@
+Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed. Note that this requires a Linux host, as Docker for Windows and Mac [doesn't support passing through USB devices](https://docs.docker.com/desktop/faqs/#can-i-pass-through-a-usb-device-to-a-container).
+
+Install Docker following the [official documentation](https://docs.docker.com/get-docker/).
+
+Once Docker is installed, you can run MVT by downloading a prebuilt MVT Docker image, or by building a Docker image yourself from the MVT source repo.
+
+### Using the prebuilt Docker image
+
+```bash
+docker pull ghcr.io/mvt-project/mvt
+```
+
+You can then run the Docker container with:
+
+```
+docker run -it ghcr.io/mvt-project/mvt
+```
+
+
+### Build and run Docker image from source
+
+```bash
+git clone https://github.com/mvt-project/mvt.git
+cd mvt
+docker build -t mvt .
+```
+
+Test if the image was created successfully:
+
+```bash
+docker run -it mvt
+```
+
+If a prompt is spawned successfully, you can close it with `exit`.
+
+## Access an iOS device from Docker
+
+On the Linux host, install and start [usbmuxd](https://github.com/libimobiledevice/usbmuxd), then connect and unlock the iOS device. The daemon exposes the device through the `/var/run/usbmuxd` socket.
+
+Bind that socket into the container to let MVT communicate with the device:
+
+```bash
+docker run -it \
+    --mount type=bind,source=/var/run/usbmuxd,target=/var/run/usbmuxd \
+    ghcr.io/mvt-project/mvt
+```
+
+If you built the image from source, replace `ghcr.io/mvt-project/mvt` with `mvt`. 

docs/index.md

@@ -6,8 +6,11 @@
 
 Mobile Verification Toolkit (MVT) is a tool to facilitate the [consensual forensic analysis](introduction.md#consensual-forensics) of Android and iOS devices, for the purpose of identifying traces of compromise.
 
+It has been developed and released by the [Amnesty International Security Lab](https://securitylab.amnesty.org) in July 2021 in the context of the [Pegasus Project](https://forbiddenstories.org/about-the-pegasus-project/) along with [a technical forensic methodology](https://www.amnesty.org/en/latest/research/2021/07/forensic-methodology-report-how-to-catch-nso-groups-pegasus/). It continues to be maintained by Amnesty International and other contributors.
+
+
 In this documentation you will find instructions on how to install and run the `mvt-ios` and `mvt-android` commands, and guidance on how to interpret the extracted results.
 
 ## Resources
 
-[:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button }
+[:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button }

docs/install.md

@@ -1,49 +1,104 @@
 # Installation
 
-Before proceeding, please note that mvt requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
+Before proceeding, please note that MVT requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
 
 ## Dependencies on Linux
 
 First install some basic dependencies that will be necessary to build all required tools:
 
 ```bash
-sudo apt install python3 python3-pip libusb-1.0-0
+sudo apt install python3 python3-venv python3-pip sqlite3 libusb-1.0-0
 ```
 
 *libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
 
-## Dependencies on Mac
+(Recommended) Set up `pipx`
 
-Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installed.
+For Ubuntu 23.04 or above:
+```bash
+sudo apt install pipx
+pipx ensurepath
+```
+
+For Ubuntu 22.04 or below:
+```
+python3 -m pip install --user pipx
+python3 -m pipx ensurepath
+```
+
+Other distributions: check for a `pipx` or `python-pipx` via your package manager.
+
+When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools). If you prefer to install a package made available by your distribution of choice, please make sure the version is recent to ensure compatibility with modern Android devices.
+
+## Dependencies on macOS
+
+Running MVT on macOS requires Xcode and [homebrew](https://brew.sh) to be installed.
 
 In order to install dependencies use:
 
 ```bash
-brew install python3 libusb
+brew install python3 pipx libusb sqlite3
 ```
 
 *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
 
-## Installing MVT
-
-If you haven't done so, you can add this to your `.bashrc` or `.zshrc` file in order to add locally installed Pypi binaries to your `$PATH`:
+When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools):
 
 ```bash
-export PATH=$PATH:~/.local/bin
+brew install --cask android-platform-tools
 ```
 
-Then you can install MVT directly from [pypi](https://pypi.org/project/mvt/)
+Or by downloading the [official binary releases](https://developer.android.com/studio/releases/platform-tools).
 
+## MVT on Windows
+
+MVT does not currently officially support running natively on Windows. While most functionality should work out of the box, there are known issues especially with `mvt-android`.
+
+It is recommended to try installing and running MVT from [Windows Subsystem Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/about) and follow Linux installation instructions for your distribution of choice.
+
+## Installing MVT
+
+### Installing from PyPI with pipx (recommended)
+1. Install `pipx` following the instructions above for your OS/distribution. Make sure to run `pipx ensurepath` and open a new terminal window.
+2. ```bash
+   pipx install mvt
+   ```
+
+You now should have the `mvt-ios` and `mvt-android` utilities installed. If you run into problems with these commands not being found, ensure you have run `pipx ensurepath` and opened a new terminal window.
+
+### Installing from PyPI directly into a virtual environment
+You can use `pipenv`, `poetry` etc. for your virtual environment, but the provided example is with the built-in `venv` tool:
+
+1. Create the virtual environment in a folder in the current directory named `env`:
+```bash
+python3 -m venv env
+```
+
+2. Activate the virtual environment:
+```bash
+source env/bin/activate
+```
+
+3. Install `mvt` into the virtual environment:
 ```bash
 pip install mvt
 ```
 
-Or from the source code:
+The `mvt-ios` and `mvt-android` utilities should now be available as commands whenever the virtual environment is active.
+
+### Installing from git source with pipx
+If you want to have the latest features in development, you can install MVT directly from the source code in git.
 
 ```bash
-git clone https://github.com/mvt-project/mvt.git
-cd mvt
-pip3 install .
+pipx install --force git+https://github.com/mvt-project/mvt.git
 ```
 
 You now should have the `mvt-ios` and `mvt-android` utilities installed.
+
+**Notes:**
+1. The `--force` flag is necessary to force the reinstallation of the package.
+2. To revert to using a PyPI version, it will be necessary to `pipx uninstall mvt` first.
+
+## Setting up command completions
+
+See ["Command completions"](command_completion.md)

docs/introduction.md

@@ -10,8 +10,24 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil
 - Generate JSON logs of extracted records, and separate JSON logs of all detected malicious traces.
 - Generate a unified chronological timeline of extracted records, along with a timeline all detected malicious traces.
 
+MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. MVT is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance.
+
+## Indicators of Compromise
+
+MVT supports using [indicators of compromise (IOCs)](https://github.com/mvt-project/mvt-indicators) to scan mobile devices for potential traces of targeting or infection by known spyware campaigns. This includes IOCs published by [Amnesty International](https://github.com/AmnestyTech/investigations/) and other research groups.
+
+!!! warning
+    Public indicators of compromise are insufficient to determine that a device is "clean", and not targeted with a particular spyware tool. Reliance on public indicators alone can miss recent forensic traces and give a false sense of security.
+
+    Reliable and comprehensive digital forensic support and triage requires access to non-public indicators, research and threat intelligence.
+
+    Such support is available to civil society through [Amnesty International's Security Lab](https://securitylab.amnesty.org/get-help/?c=mvt_docs) or [Access Now’s Digital Security Helpline](https://www.accessnow.org/help/).
+
+More information about using indicators of compromise with MVT is available in the [documentation](iocs.md).
+
+
 ## Consensual Forensics
 
 While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation.
 
-MVT's purpose is not to facilitate adversial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).
+MVT's purpose is not to facilitate adversarial forensics of non-consenting individuals' devices. The use of MVT and derivative products to extract and/or analyse data originating from devices used by individuals not consenting to the procedure is explicitly prohibited in the [license](license.md).

docs/iocs.md

@@ -0,0 +1,77 @@
+# Indicators of Compromise (IOCs)
+
+MVT uses [Structured Threat Information Expression (STIX)](https://oasis-open.github.io/cti-documentation/stix/intro.html) files to identify potential traces of compromise.
+
+These indicators of compromise are contained in a file with a particular structure of [JSON](https://en.wikipedia.org/wiki/JSON) with the `.stix2` or `.json` extensions.
+
+You can indicate a path to a STIX2 indicators file when checking iPhone backups or filesystem dumps. For example:
+
+```bash
+mvt-ios check-backup --iocs ~/ios/malware.stix2 --output /path/to/iphone/output /path/to/backup
+```
+
+Or, with data from an Android backup:
+
+```bash
+mvt-android check-backup --iocs ~/iocs/malware.stix2 /path/to/android/backup/
+```
+
+After extracting forensics data from a device, you are also able to compare it with any STIX2 file you indicate:
+
+```bash
+mvt-ios check-iocs --iocs ~/iocs/malware.stix2 /path/to/iphone/output/
+```
+
+The `--iocs` option can be invoked multiple times to let MVT import multiple STIX2 files at once. For example:
+
+```bash
+mvt-ios check-backup --iocs ~/iocs/malware1.stix --iocs ~/iocs/malware2.stix2 /path/to/backup
+```
+
+It is also possible to load STIX2 files automatically from the environment variable `MVT_STIX2`:
+
+```bash
+export MVT_STIX2="/home/user/IOC1.stix2:/home/user/IOC2.stix2"
+```
+
+## Network Access
+
+When checking URL indicators, MVT follows recognized shortened URLs with an
+HTTP `HEAD` request. URL checks are deduplicated and run concurrently, with at
+most 20 requests in progress at a time. Redirects within an individual URL
+chain are still followed sequentially. The following environment variables
+control these requests:
+
+- `MVT_NETWORK_ACCESS_ALLOWED` enables or disables network requests. It defaults
+  to `true`. Set it to `false` to prevent MVT from attempting to resolve
+  shortened URLs.
+- `MVT_NETWORK_TIMEOUT` sets the request timeout in seconds. It defaults to
+  `15`.
+
+For example, to run IOC checks without resolving shortened URLs:
+
+```bash
+MVT_NETWORK_ACCESS_ALLOWED=false mvt-ios check-iocs \
+    --iocs ~/iocs/malware.stix2 /path/to/iphone/output/
+```
+
+## STIX2 Support
+
+So far MVT implements only a subset of [STIX2 specifications](https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html):
+
+* It only supports checks for one value (such as `[domain-name:value='DOMAIN']`) and not boolean expressions over multiple comparisons
+* It only supports the following types: `domain-name:value`, `process:name`, `email-addr:value`, `file:name`, `file:path`, `file:hashes.md5`, `file:hashes.sha1`, `file:hashes.sha256`, `app:id`, `configuration-profile:id`, `android-property:name`, `url:value` (but each type will only be checked by a module if it is relevant to the type of data obtained)
+
+## Known repositories of STIX2 IOCs
+
+- The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
+    - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
+    - [Predator from Cytrox](https://citizenlab.ca/2021/12/pegasus-vs-predator-dissidents-doubly-infected-iphone-reveals-cytrox-mercenary-spyware/) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-12-16_cytrox/cytrox.stix2))
+    - [An Android Spyware Campaign Linked to a Mercenary Company](https://github.com/AmnestyTech/investigations/tree/master/2023-03-29_android_campaign) ([STIX2](https://github.com/AmnestyTech/investigations/blob/master/2023-03-29_android_campaign/malware.stix2))
+- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://raw.githubusercontent.com/Te-k/stalkerware-indicators/master/generated/stalkerware.stix2).
+- We are also maintaining [a list of IOCs](https://github.com/mvt-project/mvt-indicators) in STIX format from public spyware campaigns.
+
+You can automaticallly download the latest public indicator files with the command `mvt-ios download-iocs` or `mvt-android download-iocs`. These commands download the list of indicators from the [mvt-indicators](https://github.com/mvt-project/mvt-indicators/blob/main/indicators.yaml) repository and store them in the [appdir](https://pypi.org/project/appdirs/) folder. They are then loaded automatically by MVT.
+
+Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.
+

docs/ios/backup/check.md

@@ -2,6 +2,32 @@
 
 The backup might take some time. It is best to make sure the phone remains unlocked during the backup process. Afterwards, a new folder will be created under the path you specified using the UDID of the iPhone you backed up.
 
+## Extracting and saving the decryption key (optional)
+
+If you do not wish to enter a password every time when decrypting a backup, MVT can accept a key file instead. This key can be used with the `decrypt-backup` command.
+
+To generate a key file, you will need your device backup and the backup password:
+
+    $ mvt-ios extract-key --help
+    Usage: mvt-ios extract-key [OPTIONS] BACKUP_PATH
+
+      Extract decryption key from an iTunes backup
+
+    Options:
+      -p, --password TEXT  Password to use to decrypt the backup  [required]
+      -k, --key-file FILE  Key file to be written (if unset, will print to STDOUT)
+      --help               Show this message and exit.
+
+You can specify the password on the command line, or omit the `-p` option to have MVT prompt for a password. The `-k` option specifies where to save the file containing the decryption key. If `-k` is omitted, MVT will display the decryption key without saving.
+
+_Note_: This decryption key is sensitive data! Keep the file safe.
+
+To extract the key and have MVT prompt for a password:
+
+```bash
+mvt-ios extract-key -k /path/to/save/key /path/to/backup
+```
+
 ## Decrypting a backup
 
 In case you have an encrypted backup, you will need to decrypt it first. This can be done with `mvt-ios` as well:
@@ -15,9 +41,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca
       -d, --destination TEXT  Path to the folder where to store the decrypted
                               backup  [required]
 
-      -p, --password TEXT     Password to use to decrypt the backup NOTE: This
-                              argument is mutually exclusive with arguments:
-                              [key_file].
+      -p, --password TEXT     Password to use to decrypt the backup (or, set
+                              MVT_IOS_BACKUP_PASSWORD environment variable)
+                              NOTE: This argument is mutually exclusive with
+                              arguments: [key_file].
 
       -k, --key-file PATH     File containing raw encryption key to use to decrypt
                               the backup NOTE: This argument is mutually exclusive
@@ -25,10 +52,10 @@ In case you have an encrypted backup, you will need to decrypt it first. This ca
 
       --help                  Show this message and exit.
 
-You can specify either a password via command-line or pass a key file, and you need to specify a destination path where the decrypted backup will be stored. Following is an example usage of `decrypt-backup`:
+You can specify the password in the environment variable `MVT_IOS_BACKUP_PASSWORD`, or via command-line argument, or you can pass a key file.  You need to specify a destination path where the decrypted backup will be stored. If a password cannot be found and no key file is specified, MVT will ask for a password. Following is an example usage of `decrypt-backup` sending the password via an environment variable:
 
 ```bash
-mvt-ios decrypt-backup -p password -d /path/to/decrypted /path/to/backup
+MVT_IOS_BACKUP_PASSWORD="mypassword" mvt-ios decrypt-backup -d /path/to/decrypted /path/to/backup
 ```
 
 ## Run `mvt-ios` on a Backup

docs/ios/backup/itunes.md

@@ -1,16 +1,41 @@
 # Backup with iTunes app
 
-It is possible to do an iPhone backup by using iTunes on Windows or Mac computers (in most recent versions of Mac OS, this feature is included in Finder).
+It is possible to do an iPhone backup by using iTunes on Windows or macOS computers (in most recent versions of macOS, this feature is included in Finder, see below).
 
 To do that:
 
-* Make sure iTunes is installed.
-* Connect your iPhone to your computer using a Lightning/USB cable.
-* Open the device in iTunes (or Finder on Mac OS).
-* If you want to have a more accurate detection, ensure that the encrypted backup option is activated and choose a secure password for the backup.
-* Start the backup and wait for it to finish (this may take up to 30 minutes).
+1. Make sure iTunes is installed.
+2. Connect your iPhone to your computer using a Lightning/USB cable.
+3. Open the device in iTunes (or Finder on macOS).
+4. If you want to have a more accurate detection, ensure that the encrypted backup option is activated and choose a secure password for the backup.
+5. Start the backup and wait for it to finish (this may take up to 30 minutes).
 
-![](../../../img/macos-backup.jpg)
+![](../../img/macos-backup.jpg)
 _Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
 
-* Once the backup is done, find its location and copy it to a place where it can be analyzed by `mvt`. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On Mac OS, the backup is stored in `~/Library/Application Support/MobileSync/`.
+Once the backup is done, find its location and copy it to a place where it can be analyzed by MVT. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On macOS, the backup is stored in `~/Library/Application Support/MobileSync/`.
+
+# Backup with Finder
+
+On more recent MacOS versions, this feature is included in Finder. To do a backup:
+
+1. Launch Finder on your Mac.
+2. Connect your iPhone to your Mac using a Lightning/USB cable.
+3. Select your device from the list of devices located at the bottom of the left side bar labeled "locations".
+4. In the General tab, select `Back up all the data on your iPhone to this Mac` from the options under the Backups section.
+5. Check the box that says `Encrypt local backup`. If it is your first time selecting this option, you may need to enter a password to encrypt the backup.
+
+![](../../img/macos-backup2.png)
+_Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
+
+6. Click `Back Up Now` to start the back-up process.
+7. The encrypted backup for your iPhone should now start. Once the process finishes, you can check the backup by opening `Finder`, clicking on the `General` tab, then click on `Manage Backups`. Now you should see a list of your backups like the image below:
+
+![](../../img/macos-backups.png)
+_Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
+
+If your backup has a lock next to it like in the image above, then the backup is encrypted. You should also see the date and time when the encrypted backup was created. The backup files are stored in `~/Library/Application Support/MobileSync/`.
+
+## Notes:
+
+- Remember to keep the backup encryption password that you created safe, since without it you will not be able to access/modify/decrypt the backup file.

docs/ios/backup/libimobiledevice.md

@@ -3,10 +3,14 @@
 If you have correctly [installed libimobiledevice](../install.md) you can easily generate an iTunes backup using the `idevicebackup2` tool included in the suite. First, you might want to ensure that backup encryption is enabled (**note: encrypted backup contain more data than unencrypted backups**):
 
 ```bash
-idevicebackup2 backup encryption on
+idevicebackup2 -i encryption on
 ```
 
-Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 backup changepw` or resetting the password by resetting only the settings through the iPhone's Settings app.
+Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 -i changepw`, or by turning off encryption (`idevicebackup2 -i encryption off`) and turning it back on again.
+
+If you are not able to recover or change the password, you should try to disable encryption and obtain an unencrypted backup.
+
+If all else fails, as a *last resort* you can try resetting the password by [resetting all the settings through the iPhone's Settings app](https://support.apple.com/en-us/HT205220), via `Settings » General » Reset » Reset All Settings`.  Note that resetting the settings through the iPhone's Settings app will wipe some of the files that contain useful forensic traces, so try the options explained above first.
 
 Once ready, you can proceed performing the backup:
 

docs/ios/filesystem/dump.md

@@ -1,6 +1,6 @@
 # Dumping the filesystem
 
-While iTunes backup provide a lot of very useful databases and diagnistic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models.
+While iTunes backup provide a lot of very useful databases and diagnostic data, in some cases you might want to jailbreak the device and perform a full filesystem dump. In that case, you should take a look at [checkra1n](https://checkra.in/), which provides an easy way to obtain root on most recent iPhone models.
 
 !!! warning
     Before you checkra1n any device, make sure you take a full backup, and that you are prepared to do a full factory reset before restoring it. Even after using checkra1n's "Restore System", some traces of the jailbreak are still left on the device and [apps with anti-jailbreaks will be able to detect them](https://github.com/checkra1n/BugTracker/issues/279) and stop functioning.

docs/ios/install.md

@@ -1,6 +1,6 @@
 # Install libimobiledevice
 
-Before proceeding with doing any acquisition of iOS devices we recommend installing [libimobiledevice](https://www.libimobiledevice.org/) utilities. These utilities will become useful when extracting crash logs and generating iTunes backups. Because the utilities and its libraries are subject to frequent changes in response to new versions of iOS, you might want to consider compiling libimobiledevice utilities from sources. Otherwise, if available, you can try installing packages available in your distribution:
+Before proceeding with doing any acquisition of iOS devices we recommend installing [libimobiledevice](https://libimobiledevice.org/) utilities. These utilities will become useful when extracting crash logs and generating iTunes backups. Because the utilities and its libraries are subject to frequent changes in response to new versions of iOS, you might want to consider compiling libimobiledevice utilities from sources. Otherwise, if available, you can try installing packages available in your distribution:
 
 ```bash
 sudo apt install libimobiledevice-utils
@@ -45,10 +45,10 @@ Once the idevice tools are available you can check if everything works fine by c
 ideviceinfo
 ```
 
-This should some many details on the connected iOS device. If you are connecting the device to your laptop for the first time, it will require to unlock and enter the PIN code on the mobile device. If it complains that no device is connected and the mobile device is indeed plugged in through the USB cable, you might need to do this first, although typically the pairing is automatically done when connecting the device:
+This should show many details on the connected iOS device. If you are connecting the device to your laptop for the first time, it will require to unlock and enter the PIN code on the mobile device. If it complains that no device is connected and the mobile device is indeed plugged in through the USB cable, you might need to do this first, although typically the pairing is automatically done when connecting the device:
 
 ```bash
-sudo usbmuxd -f -d
+sudo usbmuxd -f -v
 idevicepair pair
 ```
 

docs/ios/methodology.md

@@ -6,10 +6,10 @@ Before jumping into acquiring and analyzing data from an iOS device, you should
 
 You will need to decide whether to attempt to jailbreak the device and obtain a full filesystem dump, or not.
 
-While access the full filesystem allows to extact data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again.
+While access the full filesystem allows to extract data that would otherwise be unavailable, it might not always be possible to jailbreak a certain iPhone model or version of iOS. In addition, depending on the type of jailbreak available, doing so might compromise some important records, pollute others, or potentially cause unintended malfunctioning of the device later in case it is used again.
 
 If you are not expected to return the phone, you might want to consider to attempt a jailbreak after having exhausted all other options, including a backup.
 
 #### iTunes Backup
 
-An alternative option is to generate an iTunes backup (in most recent version of mac OS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc.
+An alternative option is to generate an iTunes backup (in most recent version of macOS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc.

docs/ios/records.md

@@ -4,10 +4,44 @@ In this page you can find a (reasonably) up-to-date breakdown of the files creat
 
 ## Records extracted by `check-fs` or `check-backup`
 
+### `analytics.json`
+
+!!! info "Availability"
+    Backup (if encrypted): :material-close:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `Analytics` module. The module extracts records from the plists inside the SQLite databases located at *private/var/Keychains/Analytics/\*.db*, which contain various analytics information regarding networking, certificate-pinning, TLS, etc. failures.
+
+If indicators are provided through the command-line, processes and domains are checked against all fields of the plist. Any matches are stored in *analytics_detected.json*.
+
+---
+
+### `applications.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `Applications` module. The module extracts the list of applications installed on the device from the `Info.plist` file in backup, or from the `iTunesMetadata.plist` files in a file system dump. These records contains detailed information on the source and installation of the app.
+
+If indicators are provided through the command-line, processes and application ids are checked against the app name of each application. It also flags any applications not installed from the AppStore. Any matches are stored in *applications_detected.json*.
+
+---
+
+### `backup_info.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-close:
+
+This JSON file is created by mvt-ios' `BackupInfo` module. The module extracts some details about the backup and the device, such as name, phone number, IMEI, product type and version.
+
+---
+
 ### `cache_files.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `CacheFiles` module. The module extracts records from all SQLite database files stored on disk with the name *Cache.db*. These databases typically contain data from iOS' [internal URL caching](https://developer.apple.com/documentation/foundation/nsurlcache). Through this module you might be able to recover records of HTTP requests and responses performed my applications as well as system services, that would otherwise be unavailable. For example, you might see HTTP requests part of an exploitation chain performed by an iOS service attempting to download a first stage malicious payload.
@@ -16,10 +50,22 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
+### `calendar.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `Calendar` module. This module extracts all CalendarItems from the `Calendar.sqlitedb` database. This database contains all calendar entries from the different calendars installed on the phone.
+
+If indicators are provided through the command-line, email addresses are checked against the inviter's email of the different events. Any matches are stored in *calendar_detected.json*.
+
+---
+
 ### `calls.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Calls` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CallHistoryDB/CallHistory.storedata*, which contains records of incoming and outgoing calls, including from messaging apps such as WhatsApp or Skype.
@@ -29,7 +75,7 @@ This JSON file is created by mvt-ios' `Calls` module. The module extracts record
 ### `chrome_favicon.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `ChromeFavicon` module. The module extracts records from a SQLite database located at */private/var/mobile/Containers/Data/Application/\*/Library/Application Support/Google/Chrome/Default/Favicons*, which contains a mapping of favicons' URLs and the visited URLs which loaded them.
@@ -41,19 +87,31 @@ If indicators are provided through the command-line, they are checked against bo
 ### `chrome_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `ChromeHistory` module. The module extracts records from a SQLite database located at */private/var/mobile/Containers/Data/Application/\*/Library/Application Support/Google/Chrome/Default/History*, which contains a history of URL visits.
 
-If indicators a provided through the command-line, they are checked against the visited URL. Any matches are stored in *chrome_history_detected.json*.
+If indicators are provided through the command-line, they are checked against the visited URL. Any matches are stored in *chrome_history_detected.json*.
+
+---
+
+### `configuration_profiles.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-close:
+
+This JSON file is created by mvt-ios' `ConfigurationProfiles` module. The module extracts details about iOS configuration profiles that have been installed on the device. These should include both default iOS as well as third-party profiles.
+
+If indicators are provided through the command-line, they are checked against the configuration profile UUID to identify any known malicious profiles. Any matches are stored in *configuration_profiles_detected.json*.
 
 ---
 
 ### `contacts.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Contacts` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/AddressBook/AddressBook.sqlitedb*, which contains records from the phone's address book. While this database obviously would not contain any malicious indicators per se, you might want to use it to compare records from other apps (such as iMessage, SMS, etc.) to filter those originating from unknown origins.
@@ -63,7 +121,7 @@ This JSON file is created by mvt-ios' `Contacts` module. The module extracts rec
 ### `firefox_favicon.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `FirefoxFavicon` module. The module extracts records from a SQLite database located at */private/var/mobile/profile.profile/browser.db*, which contains a mapping of favicons' URLs and the visited URLs which loaded them.
@@ -75,29 +133,51 @@ If indicators are provided through the command-line, they are checked against bo
 ### `firefox_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `FirefoxHistory` module. The module extracts records from a SQLite database located at */private/var/mobile/profile.profile/browser.db*, which contains a history of URL visits.
 
-If indicators a provided through the command-line, they are checked against the visited URL. Any matches are stored in *firefox_history_detected.json*.
+If indicators are provided through the command-line, they are checked against the visited URL. Any matches are stored in *firefox_history_detected.json*.
+
+---
+
+### `global_preferences.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `GlobalPreferences` module. The module extracts records from a Plist file located at */private/var/mobile/Library/Preferences/.GlobalPreferences.plist*, which contains a system preferences including if Lockdown Mode is enabled.
 
 ---
 
 ### `id_status_cache.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (before iOS 14.7): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `IDStatusCache` module. The module extracts records from a plist file located at */private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist*, which contains a cache of Apple user ID authentication. This chance will indicate when apps like Facetime and iMessage first established contacts with other registered Apple IDs. This is significant because it might contain traces of malicious accounts involved in exploitation of those apps.
 
+Starting from iOS 14.7.0, this file is empty or absent.
+
+---
+
+### `shortcuts.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `Shortcuts` module. The module extracts records from an SQLite database located at */private/var/mobile/Library/Shortcuts/Shortcuts.sqlite*, which contains records about the Shortcuts application. Shortcuts are a built-in iOS feature which allows users to automation certain actions on their device. In some cases the legitimate Shortcuts app may be abused by spyware to maintain persistence on an infected devices.
+
 ---
 
 ### `interaction_c.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `InteractionC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CoreDuet/People/interactionC.db*, which contains details about user interactions with installed apps.
@@ -107,7 +187,7 @@ This JSON file is created by mvt-ios' `InteractionC` module. The module extracts
 ### `locationd_clients.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `LocationdClients` module. The module extracts records from a plist file located at */private/var/mobile/Library/Caches/locationd/clients.plist*, which contains a cache of apps which requested access to location services.
@@ -117,7 +197,7 @@ This JSON file is created by mvt-ios' `LocationdClients` module. The module extr
 ### `manifest.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-close:
 
 This JSON file is created by mvt-ios' `Manifest` module. The module extracts records from the SQLite database *Manifest.db* contained in iTunes backups, and which indexes the locally backed-up files to the original paths on the iOS device.
@@ -126,13 +206,25 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
+### `os_analytics_ad_daily.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `OSAnalyticsADDaily` module. The module extracts records from a plist located *private/var/mobile/Library/Preferences/com.apple.osanalytics.addaily.plist*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe.
+
+If indicators are provided through the command-line, they are checked against the process names. Any matches are stored in *os_analytics_ad_daily_detected.json*.
+
+---
+
 ### `datausage.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `Datausage` module. The module extracts records from a SQLite database located */private/var/wireless/Library/Databases/DataUsage.sqlite*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention.
+This JSON file is created by mvt-ios' `Datausage` module. The module extracts records from a SQLite database located */private/var/wireless/Library/Databases/DataUsage.sqlite*, which contains a history of network data usage by processes running on the system. It does not log network traffic through WiFi (the fields `WIFI_IN` and `WIFI_OUT` are always empty), and the `WWAN_IN` and `WWAN_OUT` fields are stored in bytes. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention.
 
 If indicators are provided through the command-line, they are checked against the process names. Any matches are stored in *datausage_detected.json*. If running on a full filesystem dump and if the `--fast` flag was not enabled by command-line, mvt-ios will highlight processes which look suspicious and check the presence of a binary file of the same name in the dump.
 
@@ -141,7 +233,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `netusage.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Netusage` module. The module extracts records from a SQLite database located */private/var/networkd/netusage.sqlite*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention.
@@ -150,22 +242,32 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
+### `profile_events.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-close:
+
+This JSON file is created by mvt-ios' `ProfileEvents` module. The module extracts a timeline of configuration profile operations. For example, it should indicate when a new profile was installed from the Settings app, or when one was removed.
+
+---
+
 ### `safari_browser_state.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariBrowserState` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/BrowserState.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/BrowserState.db*, which contain records of opened tabs.
 
-If indicators a provided through the command-line, they are checked against the visited URL. Any matches are stored in *safari_browser_state_detected.json*.
+If indicators are provided through the command-line, they are checked against the visited URL. Any matches are stored in *safari_browser_state_detected.json*.
 
 ---
 
 ### `safari_favicon.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariFavicon` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Image Cache/Favicons/Favicons.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Image Cache/Favicons/Favicons.db*, which contain mappings of favicons' URLs and the visited URLs which loaded them.
@@ -177,7 +279,7 @@ If indicators are provided through the command-line, they are checked against bo
 ### `safari_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariHistory` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/History.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/History.db*, which contain a history of URL visits.
@@ -186,32 +288,63 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
+### `shutdown_log.json`
+
+!!! info "Availability"
+    Backup (if encrypted): :material-close:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `ShutdownLog` module. The module extracts records from the shutdown log located at *private/var/db/diagnostics/shutdown.log*. When shutting down an iPhone, a SIGTERM will be sent to all processes runnning. The `shutdown.log` file will log any process (with its pid and path) that did not shut down after the SIGTERM was sent.
+
+If indicators are provided through the command-line, they are checked against the paths. Any matches are stored in *shutdown_log_detected.json*.
+
+---
+
 ### `sms.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `SMS` module. The module extracts a list of SMS messages containing HTTP links from the SQLite database located at */private/var/mobile/Library/SMS/sms.db*.
+This JSON file is created by mvt-ios' `SMS` module. The module extracts a list of SMS messages from the SQLite database located at */private/var/mobile/Library/SMS/sms.db*.
 
 If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *sms_detected.json*.
 
 ---
 
+### `urls.json`
+
+This JSON file collects URLs extracted from SMS, iMessage, and WhatsApp
+messages. Each entry contains the original URL, its expanded destination when
+MVT resolved a shortened URL during indicator checking, the message timestamp,
+and its `sms` or `whatsapp` source.
+
+---
+
 ### `sms_attachments.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SMSAttachments` module. The module extracts details about attachments sent via SMS or iMessage from the same database used by the `SMS` module. These records might be useful to indicate unique patterns that might be indicative of exploitation attempts leveraging potential vulnerabilities in file format parsers or other forms of file handling by the Messages app.
 
 ---
 
+### `tcc.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `TCC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/TCC/TCC.db*, which contains a list of which services such as microphone, camera, or location, apps have been granted or denied access to.
+
+---
+
 ### `version_history.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `IOSVersionHistory` module. The module extracts records of iOS software updates from analytics plist files located at */private/var/db/analyticsd/Analytics-Journal-\*.ips*.
@@ -221,7 +354,7 @@ This JSON file is created by mvt-ios' `IOSVersionHistory` module. The module ext
 ### `webkit_indexeddb.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitIndexedDB` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/IndexedDB*, which contains IndexedDB files created by any app installed on the device.
@@ -233,19 +366,31 @@ If indicators are provided through the command-line, they are checked against th
 ### `webkit_local_storage.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a lsit of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
+This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
 
 If indicators are provided through the command-line, they are checked against the extracted names. Any matches are stored in *webkit_local_storage_detected.json*.
 
 ---
 
+### `webkit_resource_load_statistics.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios `WebkitResourceLoadStatistics` module. The module extracts records from available WebKit ResourceLoadStatistics *observations.db* SQLite3 databases. These records should indicate domain names contacted by apps, including a timestamp.
+
+If indicators are provided through the command-line, they are checked against the extracted domain names. Any matches are stored in *webkit_resource_load_statistics_detected.json*.
+
+---
+
 ### `webkit_safari_view_service.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitSafariViewService` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/*, which contains files cached by SafariVewService.
@@ -257,7 +402,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `webkit_session_resource_log.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitSessionResourceLog` module. The module extracts records from plist files with the name *full_browsing_session_resourceLog.plist*, which contain records of resources loaded by different domains visited.
@@ -269,10 +414,24 @@ If indicators are provided through the command-line, they are checked against th
 ### `whatsapp.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages containing HTTP links from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*.
+This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*, along with one record per chat session (marked with `"record_type": "chat_session"`) containing the first and last interaction dates of each conversation. Chat sessions produce `chat_first_message` and `chat_last_message` timeline events, and group chats additionally produce a `group_created` event. A chat session's last-message date can postdate its newest stored message when the most recent messages in the chat were deleted.
+
+Recent WhatsApp versions key 1:1 chat sessions by an opaque LID identifier rather than the contact's phone number. The module resolves these using the `ZWAPHONENUMBERLIDPAIR` table from the *LID.sqlite* database in the same app group (or from *ChatStorage.sqlite* itself in versions that store it there), populating `partner_resolved_phone_number` on chat session records and using the phone number in timeline events. Each LID-phone number pair is also extracted as a record (`"record_type": "lid_phone_number_pair"`) and produces a `lid_pair_recorded` timeline event marking when WhatsApp learned the association.
 
 If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*.
 
+---
+
+### `whatsapp_contacts.json`
+
+!!! info "Availability"
+    Backup: :material-check:
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `WhatsappContacts` module. The module extracts WhatsApp contact records from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ContactsV2.sqlite*, including each contact's phone number, WhatsApp and LID identifiers, and the per-contact disappearing messages timer, which is not recorded in *ChatStorage.sqlite*. Each timestamp stored on a contact record produces a timeline event: `disappearing_mode_set` (when the disappearing messages timer was last changed), `about_changed` (when the contact last changed their "about" text), `about_expiration` (when a timed "about" is scheduled to expire) and `contact_last_updated` (when the contact record was last updated).
+
+This database is often missing from incremental backups. When it cannot be found, the module logs a warning and produces no results, in which case the disappearing messages state of chats cannot be determined from the backup.
+

docs/ios/sysdiagnose.md

@@ -0,0 +1,50 @@
+# Check an iOS Sysdiagnose
+
+`mvt-ios check-sysdiagnose` prepares an iOS sysdiagnose archive for analysis by
+custom MVT modules. MVT does not include built-in sysdiagnose modules. You must
+load at least one custom module that explicitly supports this command.
+
+The command accepts either an extracted sysdiagnose directory or the original
+gzip-compressed tar archive.
+
+```bash
+mvt-ios check-sysdiagnose \
+    --load-module ./sysdiagnose_modules.py \
+    --output ./results \
+    ./sysdiagnose_2024.01.02_03-04-05+0200.tar.gz
+```
+
+Use `--hashes` to include hashes for analyzed files in `info.json`, and
+`--list-modules` to display the eligible custom modules without running them.
+
+## Writing a custom module
+
+Extend `SysdiagnoseExtraction` to access the archive contents consistently for
+both directory and tar inputs. Each module must declare the command explicitly
+in `supported_commands`.
+
+```python
+from mvt.ios.modules.sysdiagnose import SysdiagnoseExtraction
+
+
+class ExampleSysdiagnoseModule(SysdiagnoseExtraction):
+    supported_commands = (("ios", "check-sysdiagnose"),)
+    slug = "example_sysdiagnose"
+
+    def run(self):
+        paths = self._get_files_by_pattern("*/example.log")
+        if paths:
+            content = self._get_file_content(paths[0]).decode("utf-8", "replace")
+            self.results = [{"content": content}]
+
+    def check_indicators(self):
+        pass
+
+    def serialize(self, result):
+        return None
+```
+
+The base class provides `from_sysdiagnose_folder()` and
+`from_sysdiagnose_tar()` setup hooks, as well as protected file lookup, file
+reading, and timezone extraction helpers. IPS crash-report metadata is exposed
+on `ips_files`.

docs/requirements.txt

@@ -1,5 +0,0 @@
-mkdocs==1.2.1
-mkdocs-autorefs
-mkdocs-material
-mkdocs-material-extensions
-mkdocstrings

mkdocs.yml

@@ -1,14 +1,16 @@
 site_name: Mobile Verification Toolkit
 repo_url: https://github.com/mvt-project/mvt
 edit_uri: edit/main/docs/
-copyright: Copyright © 2021 MVT Project Developers
+copyright: Copyright © 2021-2023 MVT Project Developers
 site_description: Mobile Verification Toolkit Documentation
+not_in_nav: |
+    android/adb.md
 markdown_extensions:
     - attr_list
     - admonition
     - pymdownx.emoji:
-        emoji_index: !!python/name:materialx.emoji.twemoji
-        emoji_generator: !!python/name:materialx.emoji.to_svg
+        emoji_index: !!python/name:material.extensions.emoji.twemoji
+        emoji_generator: !!python/name:material.extensions.emoji.to_svg
     - pymdownx.superfences
     - pymdownx.inlinehilite
     - pymdownx.highlight:
@@ -28,6 +30,8 @@ nav:
     - Welcome: "index.md"
     - Introduction: "introduction.md"
     - Installation: "install.md"
+    - Command Completion: "command_completion.md"
+    - Using Docker: "docker.md"
     - MVT for iOS:
         - iOS Forensic Methodology: "ios/methodology.md"
         - Install libimobiledevice: "ios/install.md"
@@ -38,9 +42,15 @@ nav:
         - Check a Filesystem Dump:
             - Dumping the filesystem: "ios/filesystem/dump.md"
             - Check a Filesystem Dump with mvt-ios: "ios/filesystem/check.md"
+        - Check a Sysdiagnose: "ios/sysdiagnose.md"
         - Records extracted by mvt-ios: "ios/records.md"
     - MVT for Android:
         - Android Forensic Methodology: "android/methodology.md"
-        - Check APKs: "android/download_apks.md"
-        - Check an Android Backup: "android/backup.md"
+        - Check an Android Backup (SMS messages): "android/backup.md"
+        - Check Android Intrusion Logs: "android/intrusion_logs.md"
+    - Indicators of Compromise: "iocs.md"
+    - Development:
+        - Development Instructions: "development/index.md"
+        - Custom CLI Commands: "development/custom_commands.md"
+        - Plugin Configuration: "development/plugin_configuration.md"
     - License: "license.md"

mvt/__init__.py

@@ -1,4 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE

mvt/android/__init__.py

@@ -1,6 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .cli import cli

mvt/android/cli.py

@@ -1,156 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sys
-import click
-import argparse
-import logging
-from rich.logging import RichHandler
-
-from mvt.common.module import run_module, save_timeline
-from mvt.common.indicators import Indicators
-from .download_apks import DownloadAPKs
-from .lookups.koodous import koodous_lookup
-from .lookups.virustotal import virustotal_lookup
-from .modules.adb import ADB_MODULES
-from .modules.backup import BACKUP_MODULES
-
-# Setup logging using Rich.
-LOG_FORMAT = "[%(name)s] %(message)s"
-logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
-    RichHandler(show_path=False, log_time_format="%X")])
-log = logging.getLogger(__name__)
-
-# Help messages of repeating options.
-OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results"
-
-
-#==============================================================================
-# Main
-#==============================================================================
-@click.group(invoke_without_command=False)
-def cli():
-    return
-
-
-#==============================================================================
-# Download APKs
-#==============================================================================
-@cli.command("download-apks", help="Download all or non-safelisted installed APKs installed on the device")
-@click.option("--all-apks", "-a", is_flag=True,
-              help="Extract all packages installed on the phone, even those marked as safe")
-@click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal")
-@click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous")
-@click.option("--all-checks", "-A", is_flag=True, help="Run all available checks")
-@click.option("--output", "-o", type=click.Path(exists=True),
-              help="Specify a path to a folder where you want to store JSON results")
-@click.option("--from-file", "-f", type=click.Path(exists=True),
-              help="Instead of acquiring from phone, load an existing packages.json file for lookups (mainly for debug purposes)")
-def download_apks(all_apks, virustotal, koodous, all_checks, output, from_file):
-    try:
-        if from_file:
-            download = DownloadAPKs.from_json(from_file)
-        else:
-            if not output:
-                log.critical("You need to specify an output folder (with --output, -o) when extracting APKs from a device")
-                sys.exit(-1)
-
-            download = DownloadAPKs(output_folder=output, all_apks=all_apks)
-            download.run()
-
-        packages = download.packages
-
-        if len(packages) == 0:
-            return
-
-        if virustotal or all_checks:
-            virustotal_lookup(packages)
-
-        if koodous or all_checks:
-            koodous_lookup(packages)
-    except KeyboardInterrupt:
-        print("")
-        sys.exit(-1)
-
-
-#==============================================================================
-# Checks through ADB
-#==============================================================================
-@cli.command("check-adb", help="Check an Android device over adb")
-@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True),
-              help="Specify a path to a folder where you want to store JSON results")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
-def check_adb(iocs, output, list_modules, module):
-    if list_modules:
-        log.info("Following is the list of available check-adb modules:")
-        for adb_module in ADB_MODULES:
-            log.info(" - %s", adb_module.__name__)
-
-        return
-
-    log.info("Checking Android through adb bridge")
-
-    if iocs:
-        # Pre-load indicators for performance reasons.
-        log.info("Loading indicators from provided file at %s", iocs)
-        indicators = Indicators(iocs)
-
-    timeline = []
-    timeline_detected = []
-    for adb_module in ADB_MODULES:
-        if module and adb_module.__name__ != module:
-            continue
-
-        m = adb_module(output_folder=output, log=logging.getLogger(adb_module.__module__))
-
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
-        run_module(m)
-        timeline.extend(m.timeline)
-        timeline_detected.extend(m.timeline_detected)
-
-    if output:
-        if len(timeline) > 0:
-            save_timeline(timeline, os.path.join(output, "timeline.csv"))
-        if len(timeline_detected) > 0:
-            save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv"))
-
-#==============================================================================
-# Check ADB backup
-#==============================================================================
-@cli.command("check-backup", help="Check an Android Backup")
-@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE)
-@click.argument("BACKUP_PATH", type=click.Path(exists=True))
-def check_backup(iocs, output, backup_path):
-    log.info("Checking ADB backup located at: %s", backup_path)
-
-    if iocs:
-        # Pre-load indicators for performance reasons.
-        log.info("Loading indicators from provided file at %s", iocs)
-        indicators = Indicators(iocs)
-
-    if os.path.isfile(backup_path):
-        log.critical("The path you specified is a not a folder!")
-
-        if os.path.basename(backup_path) == "backup.ab":
-            log.info("You can use ABE (https://github.com/nelenkov/android-backup-extractor) " \
-                     "to extract 'backup.ab' files!")
-        sys.exit(-1)
-
-    for module in BACKUP_MODULES:
-        m = module(base_folder=backup_path, output_folder=output,
-                   log=logging.getLogger(module.__module__))
-
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
-        run_module(m)

mvt/android/data/root_binaries.txt

@@ -1,10 +0,0 @@
-su
-busybox
-supersu
-Superuser.apk
-KingoUser.apk
-SuperSu.apk
-magisk
-magiskhide
-magiskinit
-magiskpolicy

mvt/android/data/root_packages.txt

@@ -1,25 +0,0 @@
-com.noshufou.android.su
-com.noshufou.android.su.elite
-eu.chainfire.supersu
-com.koushikdutta.superuser
-com.thirdparty.superuser
-com.yellowes.su
-com.koushikdutta.rommanager
-com.koushikdutta.rommanager.license
-com.dimonvideo.luckypatcher
-com.chelpus.lackypatch
-com.ramdroid.appquarantine
-com.ramdroid.appquarantinepro
-com.devadvance.rootcloak
-com.devadvance.rootcloakplus
-de.robv.android.xposed.installer
-com.saurik.substrate
-com.zachspong.temprootremovejb
-com.amphoras.hidemyroot
-com.amphoras.hidemyrootadfree
-com.formyhm.hiderootPremium
-com.formyhm.hideroot
-me.phh.superuser
-eu.chainfire.supersu.pro
-com.kingouser.com
-com.topjohnwu.magisk

mvt/android/data/safe_packages.txt

@@ -1,182 +0,0 @@
-android
-android.auto_generated_rro__
-android.autoinstalls.config.google.nexus
-com.android.backupconfirm
-com.android.bips
-com.android.bluetooth
-com.android.bluetoothmidiservice
-com.android.bookmarkprovider
-com.android.calllogbackup
-com.android.captiveportallogin
-com.android.carrierconfig
-com.android.carrierdefaultapp
-com.android.cellbroadcastreceiver
-com.android.certinstaller
-com.android.chrome
-com.android.companiondevicemanager
-com.android.connectivity.metrics
-com.android.cts.ctsshim
-com.android.cts.priv.ctsshim
-com.android.defcontainer
-com.android.documentsui
-com.android.dreams.basic
-com.android.egg
-com.android.emergency
-com.android.externalstorage
-com.android.facelock
-com.android.hotwordenrollment
-com.android.hotwordenrollment.okgoogle
-com.android.hotwordenrollment.tgoogle
-com.android.hotwordenrollment.xgoogle
-com.android.htmlviewer
-com.android.inputdevices
-com.android.keychain
-com.android.location.fused
-com.android.managedprovisioning
-com.android.mms.service
-com.android.mtp
-com.android.musicfx
-com.android.nfc
-com.android.omadm.service
-com.android.pacprocessor
-com.android.phone
-com.android.printspooler
-com.android.providers.blockednumber
-com.android.providers.calendar
-com.android.providers.contacts
-com.android.providers.downloads
-com.android.providers.downloads.ui
-com.android.providers.media
-com.android.providers.partnerbookmarks
-com.android.providers.settings
-com.android.providers.telephony
-com.android.providers.userdictionary
-com.android.proxyhandler
-com.android.retaildemo
-com.android.safetyregulatoryinfo
-com.android.sdm.plugins.connmo
-com.android.sdm.plugins.dcmo
-com.android.sdm.plugins.diagmon
-com.android.sdm.plugins.sprintdm
-com.android.server.telecom
-com.android.service.ims
-com.android.service.ims.presence
-com.android.settings
-com.android.sharedstoragebackup
-com.android.shell
-com.android.statementservice
-com.android.stk
-com.android.systemui
-com.android.systemui.theme.dark
-com.android.vending
-com.android.vpndialogs
-com.android.vzwomatrigger
-com.android.wallpaperbackup
-com.android.wallpaper.livepicker
-com.breel.wallpapers
-com.customermobile.preload.vzw
-com.google.android.apps.cloudprint
-com.google.android.apps.docs
-com.google.android.apps.docs.editors.docs
-com.google.android.apps.enterprise.dmagent
-com.google.android.apps.gcs
-com.google.android.apps.helprtc
-com.google.android.apps.inputmethod.hindi
-com.google.android.apps.maps
-com.google.android.apps.messaging
-com.google.android.apps.nexuslauncher
-com.google.android.apps.photos
-com.google.android.apps.pixelmigrate
-com.google.android.apps.tachyon
-com.google.android.apps.turbo
-com.google.android.apps.tycho
-com.google.android.apps.wallpaper
-com.google.android.apps.wallpaper.nexus
-com.google.android.apps.work.oobconfig
-com.google.android.apps.youtube.vr
-com.google.android.asdiv
-com.google.android.backuptransport
-com.google.android.calculator
-com.google.android.calendar
-com.google.android.carrier
-com.google.android.carrier.authdialog
-com.google.android.carrierentitlement
-com.google.android.carriersetup
-com.google.android.configupdater
-com.google.android.contacts
-com.google.android.deskclock
-com.google.android.dialer
-com.google.android.euicc
-com.google.android.ext.services
-com.google.android.ext.shared
-com.google.android.feedback
-com.google.android.gm
-com.google.android.gms
-com.google.android.gms.policy_auth
-com.google.android.gms.policy_sidecar_o
-com.google.android.gms.setup
-com.google.android.GoogleCamera
-com.google.android.googlequicksearchbox
-com.google.android.gsf
-com.google.android.gsf.login
-com.google.android.hardwareinfo
-com.google.android.hiddenmenu
-com.google.android.ims
-com.google.android.inputmethod.japanese
-com.google.android.inputmethod.korean
-com.google.android.inputmethod.latin
-com.google.android.inputmethod.pinyin
-com.google.android.instantapps.supervisor
-com.google.android.keep
-com.google.android.marvin.talkback
-com.google.android.music
-com.google.android.nexusicons
-com.google.android.onetimeinitializer
-com.google.android.packageinstaller
-com.google.android.partnersetup
-com.google.android.printservice.recommendation
-com.google.android.setupwizard
-com.google.android.soundpicker
-com.google.android.storagemanager
-com.google.android.syncadapters.contacts
-com.google.android.tag
-com.google.android.talk
-com.google.android.tetheringentitlement
-com.google.android.theme.pixel
-com.google.android.tts
-com.google.android.videos
-com.google.android.vr.home
-com.google.android.vr.inputmethod
-com.google.android.webview
-com.google.android.wfcactivation
-com.google.android.youtube
-com.google.ar.core
-com.google.intelligence.sense
-com.google.modemservice
-com.google.pixel.wahoo.gfxdrv
-com.google.SSRestartDetector
-com.google.tango
-com.google.vr.apps.ornament
-com.google.vr.vrcore
-com.htc.omadm.trigger
-com.qti.qualcomm.datastatusnotification
-com.qualcomm.atfwd
-com.qualcomm.embms
-com.qualcomm.fastdormancy
-com.qualcomm.ltebc_vzw
-com.qualcomm.qcrilmsgtunnel
-com.qualcomm.qti.ims
-com.qualcomm.qti.networksetting
-com.qualcomm.qti.telephonyservice
-com.qualcomm.qti.uceShimService
-com.qualcomm.shutdownlistner
-com.qualcomm.timeservice
-com.qualcomm.vzw_api
-com.quicinc.cne.CNEService
-com.verizon.llkagent
-com.verizon.mips.services
-com.verizon.obdm
-com.verizon.obdm_permissions
-com.verizon.services
-com.vzw.apnlib
-qualcomm.com.vzw_msdc_api

mvt/android/download_apks.py

@@ -1,212 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import json
-import logging
-import pkg_resources
-from tqdm import tqdm
-
-from mvt.common.utils import get_sha256_from_file_path
-from .modules.adb.base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-# TODO: Would be better to replace tqdm with rich.progress to reduce
-#       the number of dependencies. Need to investigate whether
-#       it's possible to have a simialr callback system.
-class PullProgress(tqdm):
-    """PullProgress is a tqdm update system for APK downloads."""
-
-    def update_to(self, file_name, current, total):
-        if total is not None:
-            self.total = total
-        self.update(current - self.n)
-
-
-class Package:
-    """Package indicates a package name and all the files associated with it."""
-
-    def __init__(self, name, files=None):
-        self.name = name
-        self.files = files or []
-
-
-class DownloadAPKs(AndroidExtraction):
-    """DownloadAPKs is the main class operating the download of APKs
-    from the device."""
-
-    def __init__(self, output_folder=None, all_apks=False, packages=None):
-        """Initialize module.
-        :param output_folder: Path to the folder where data should be stored
-        :param all_apks: Boolean indicating whether to download all packages
-                         or filter known-goods 
-        :param packages: Provided list of packages, typically for JSON checks
-        """
-        super().__init__(file_path=None, base_folder=None,
-                         output_folder=output_folder)
-
-        self.output_folder_apk = None
-        self.packages = packages or []
-        self.all_apks = all_apks
-
-        self._safe_packages = []
-
-    @classmethod
-    def from_json(cls, json_path):
-        """Initialize this class from an existing packages.json file.
-        :param json_path: Path to the packages.json file to parse.
-        """
-        with open(json_path, "r") as handle:
-            data = json.load(handle)
-
-            packages = []
-            for entry in data:
-                package = Package(entry["name"], entry["files"])
-                packages.append(package)
-
-            return cls(packages=packages)
-
-    def _load_safe_packages(self):
-        """Load known-good package names.
-        """
-        safe_packages_path = os.path.join("data", "safe_packages.txt")
-        safe_packages_string = pkg_resources.resource_string(__name__, safe_packages_path)
-        safe_packages_list = safe_packages_string.decode("utf-8").split("\n")
-        self._safe_packages.extend(safe_packages_list)
-
-    def _clean_output(self, output):
-        """Clean adb shell command output.
-        :param output: Command output to clean.
-        """
-        return output.strip().replace("package:", "")
-
-    def get_packages(self):
-        """Retrieve package names from the device using adb.
-        """
-        log.info("Retrieving package names ...")
-
-        if not self.all_apks:
-            self._load_safe_packages()
-
-        output = self._adb_command("pm list packages")
-        total = 0
-        for line in output.split("\n"):
-            package_name = self._clean_output(line)
-            if package_name == "":
-                continue
-
-            total += 1
-
-            if not self.all_apks and package_name in self._safe_packages:
-                continue
-
-            if package_name not in self.packages:
-                self.packages.append(Package(package_name))
-
-        log.info("There are %d packages installed on the device. I selected %d for inspection.",
-                 total, len(self.packages))
-
-    def pull_package_file(self, package_name, remote_path):
-        """Pull files related to specific package from the device.
-        :param package_name: Name of the package to download
-        :param remote_path: Path to the file to download
-        :returns: Path to the local copy
-        """
-        log.info("Downloading %s ...", remote_path)
-
-        file_name = ""
-        if "==/" in remote_path:
-            file_name = "_" + remote_path.split("==/")[1].replace(".apk", "")
-
-        local_path = os.path.join(self.output_folder_apk,
-                                  f"{package_name}{file_name}.apk")
-        name_counter = 0
-        while True:
-            if not os.path.exists(local_path):
-                break
-
-            name_counter += 1
-            local_path = os.path.join(self.output_folder_apk,
-                                      f"{package_name}{file_name}_{name_counter}.apk")
-
-        try:
-            with PullProgress(unit='B', unit_divisor=1024, unit_scale=True,
-                              miniters=1) as pp:
-                self._adb_download(remote_path, local_path,
-                                   progress_callback=pp.update_to)
-        except Exception as e:
-            log.exception("Failed to pull package file from %s: %s",
-                          remote_path, e)
-            self._adb_reconnect()
-            return None
-
-        return local_path
-
-    def pull_packages(self):
-        """Download all files of all selected packages from the device.
-        """
-        log.info("Starting extraction of installed APKs at folder %s", self.output_folder)
-
-        if not os.path.exists(self.output_folder):
-            os.mkdir(self.output_folder)
-
-        log.info("Downloading packages from device. This might take some time ...")
-
-        self.output_folder_apk = os.path.join(self.output_folder, "apks")
-        if not os.path.exists(self.output_folder_apk):
-            os.mkdir(self.output_folder_apk)
-
-        total_packages = len(self.packages)
-        counter = 0
-        for package in self.packages:
-            counter += 1
-
-            log.info("[%d/%d] Package: %s", counter, total_packages, package.name)
-
-            try:
-                output = self._adb_command(f"pm path {package.name}")
-                output = self._clean_output(output)
-                if not output:
-                    continue
-            except Exception as e:
-                log.exception("Failed to get path of package %s: %s", package.name, e)
-                self._adb_reconnect()
-                continue
-
-            # Sometimes the package path contains multiple lines for multiple apks.
-            # We loop through each line and download each file.
-            for path in output.split("\n"):
-                device_path = path.strip()
-                file_path = self.pull_package_file(package.name, device_path)
-                if not file_path:
-                    continue
-
-                # We add the apk metadata to the package object.
-                package.files.append({
-                    "path": device_path,
-                    "local_name": file_path,
-                    "sha256": get_sha256_from_file_path(file_path),
-                })
-
-    def save_json(self):
-        """Save the results to the package.json file.
-        """
-        json_path = os.path.join(self.output_folder, "packages.json")
-        packages = []
-        for package in self.packages:
-            packages.append(package.__dict__)
-
-        with open(json_path, "w") as handle:
-            json.dump(packages, handle, indent=4)
-
-    def run(self):
-        """Run all steps of fetch-apk.
-        """
-        self._adb_connect()
-        self.get_packages()
-        self.pull_packages()
-        self.save_json()
-        self._adb_disconnect()

mvt/android/lookups/__init__.py

@@ -1,4 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE

mvt/android/lookups/koodous.py

@@ -1,57 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import requests
-import logging
-
-from rich.text import Text
-from rich.table import Table
-from rich.progress import track
-from rich.console import Console
-
-log = logging.getLogger(__name__)
-
-def koodous_lookup(packages):
-    log.info("Looking up all extracted files on Koodous (www.koodous.com)")
-    log.info("This might take a while...")
-
-    table = Table(title="Koodous Packages Detections")
-    table.add_column("Package name")
-    table.add_column("File name")
-    table.add_column("Trusted")
-    table.add_column("Detected")
-    table.add_column("Rating")
-
-    total_packages = len(packages)
-    for i in track(range(total_packages), description=f"Looking up {total_packages} packages..."):
-        package = packages[i]
-        for file in package.files:
-            url = f"https://api.koodous.com/apks/{file['sha256']}"
-            res = requests.get(url)
-            report = res.json()
-
-            row = [package.name, file["local_name"]]
-
-            if "package_name" in report:
-                trusted = "no"
-                if report["trusted"]:
-                    trusted = Text("yes", "green bold")
-
-                detected = "no"
-                if report["detected"]:
-                    detected = Text("yes", "red bold")
-
-                rating = "0"
-                if int(report["rating"]) < 0:
-                    rating = Text(str(report["rating"]), "red bold")
-
-                row.extend([trusted, detected, rating])
-            else:
-                row.extend(["n/a", "n/a", "n/a"])
-
-            table.add_row(*row)
-
-    console = Console()
-    console.print(table)

mvt/android/lookups/virustotal.py

@@ -1,92 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import requests
-import logging
-from rich.text import Text
-from rich.table import Table
-from rich.progress import track
-from rich.console import Console
-
-log = logging.getLogger(__name__)
-
-def get_virustotal_report(hashes):
-    apikey = "233f22e200ca5822bd91103043ccac138b910db79f29af5616a9afe8b6f215ad"
-    url = f"https://www.virustotal.com/partners/sysinternals/file-reports?apikey={apikey}"
-
-    items = []
-    for sha256 in hashes:
-        items.append({
-            "autostart_location": "",
-            "autostart_entry": "",
-            "hash": sha256,
-            "local_name": "",
-            "creation_datetime": "",
-        })
-    headers = {"User-Agent": "VirusTotal", "Content-Type": "application/json"}
-    res = requests.post(url, headers=headers, json=items)
-
-    if res.status_code == 200:
-        report = res.json()
-        return report["data"]
-    else:
-        log.error("Unexpected response from VirusTotal: %s", res.status_code)
-        return None
-
-def virustotal_lookup(packages):
-    log.info("Looking up all extracted files on VirusTotal (www.virustotal.com)")
-
-    unique_hashes = []
-    for package in packages:
-        for file in package.files:
-            if file["sha256"] not in unique_hashes:
-                unique_hashes.append(file["sha256"])
-
-    total_unique_hashes = len(unique_hashes)
-
-    detections = {}
-    def virustotal_query(batch):
-        report = get_virustotal_report(batch)
-        if not report:
-            return
-
-        for entry in report:
-            if entry["hash"] not in detections and entry["found"] is True:
-                detections[entry["hash"]] = entry["detection_ratio"]
-
-    batch = []
-    for i in track(range(total_unique_hashes), description=f"Looking up {total_unique_hashes} files..."):
-        file_hash = unique_hashes[i]
-        batch.append(file_hash)
-        if len(batch) == 25:
-            virustotal_query(batch)
-            batch = []
-
-    if batch:
-        virustotal_query(batch)
-
-    table = Table(title="VirusTotal Packages Detections")
-    table.add_column("Package name")
-    table.add_column("File path")
-    table.add_column("Detections")
-
-    for package in packages:
-        for file in package.files:
-            row = [package.name, file["local_name"]]
-
-            if file["sha256"] in detections:
-                detection = detections[file["sha256"]]
-                positives = detection.split("/")[0]
-                if int(positives) > 0:
-                    row.append(Text(detection, "red bold"))
-                else:
-                    row.append(detection)
-            else:
-                row.append("not found")
-
-            table.add_row(*row)
-
-    console = Console()
-    console.print(table)

mvt/android/modules/__init__.py

@@ -1,4 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE

mvt/android/modules/adb/__init__.py

@@ -1,18 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .chrome_history import ChromeHistory
-from .dumpsys_batterystats import DumpsysBatterystats
-from .dumpsys_packages import DumpsysPackages
-from .dumpsys_procstats import DumpsysProcstats
-from .processes import Processes
-from .sms import SMS
-from .whatsapp import Whatsapp
-from .packages import Packages
-from .rootbinaries import RootBinaries
-
-ADB_MODULES = [ChromeHistory, SMS, Whatsapp, Processes,
-               DumpsysBatterystats, DumpsysProcstats,
-               DumpsysPackages, Packages, RootBinaries]

mvt/android/modules/adb/base.py

@@ -1,166 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sys
-import time
-import logging
-import tempfile
-from adb_shell.adb_device import AdbDeviceUsb
-from adb_shell.auth.keygen import keygen, write_public_keyfile
-from adb_shell.auth.sign_pythonrsa import PythonRSASigner
-from adb_shell.exceptions import DeviceAuthError, AdbCommandFailureException
-from usb1 import USBErrorBusy, USBErrorAccess
-
-from mvt.common.module import MVTModule
-
-log = logging.getLogger(__name__)
-
-ADB_KEY_PATH = os.path.expanduser("~/.android/adbkey")
-ADB_PUB_KEY_PATH = os.path.expanduser("~/.android/adbkey.pub")
-
-class AndroidExtraction(MVTModule):
-    """This class provides a base for all Android extraction modules."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        """Initialize Android extraction module.
-        :param file_path: Path to the database file to parse
-        :param base_folder: Path to a base folder containing an Android dump
-        :param output_folder: Path to the folder where to store extraction
-                              results
-        """
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-        self.device = None
-
-    def _adb_check_keys(self):
-        """Make sure Android adb keys exist.
-        """
-        if not os.path.exists(ADB_KEY_PATH):
-            keygen(ADB_KEY_PATH)
-
-        if not os.path.exists(ADB_PUB_KEY_PATH):
-            write_public_keyfile(ADB_KEY_PATH, ADB_PUB_KEY_PATH)
-
-    def _adb_connect(self):
-        """Connect to the device over adb.
-        """
-        self._adb_check_keys()
-
-        with open(ADB_KEY_PATH, "rb") as handle:
-            priv_key = handle.read()
-
-        signer = PythonRSASigner("", priv_key)
-        self.device = AdbDeviceUsb()
-
-        while True:
-            try:
-                self.device.connect(rsa_keys=[signer], auth_timeout_s=5)
-            except (USBErrorBusy, USBErrorAccess):
-                log.critical("Device is busy, maybe run `adb kill-server` and try again.")
-                sys.exit(-1)
-            except DeviceAuthError:
-                log.error("You need to authorize this computer on the Android device. Retrying in 5 seconds...")
-                time.sleep(5)
-            except Exception as e:
-                log.critical(e)
-                sys.exit(-1)
-            else:
-                break
-
-    def _adb_disconnect(self):
-        """Close adb connection to the device.
-        """
-        self.device.close()
-
-    def _adb_reconnect(self):
-        """Reconnect to device using adb.
-        """
-        log.info("Reconnecting ...")
-        self._adb_disconnect()
-        self._adb_connect()
-
-    def _adb_command(self, command):
-        """Execute an adb shell command.
-        :param command: Shell command to execute
-        :returns: Output of command
-        """
-        return self.device.shell(command)
-
-    def _adb_check_if_root(self):
-        """Check if we have a `su` binary on the Android device.
-        :returns: Boolean indicating whether a `su` binary is present or not
-        """
-        return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1"))
-
-    def _adb_root_or_die(self):
-        """Check if we have a `su` binary, otherwise raise an Exception.
-        """
-        if not self._adb_check_if_root():
-            raise Exception("The Android device does not seem to have a `su` binary. Cannot run this module.")
-
-    def _adb_command_as_root(self, command):
-        """Execute an adb shell command.
-        :param command: Shell command to execute as root
-        :returns: Output of command
-        """
-        return self._adb_command(f"su -c {command}")
-
-    def _adb_download(self, remote_path, local_path, progress_callback=None):
-        """Download a file form the device.
-        :param remote_path: Path to download from the device
-        :param local_path: Path to where to locally store the copy of the file
-        :param progress_callback: Callback for download progress bar
-        """
-        try:
-            self.device.pull(remote_path, local_path, progress_callback)
-        except AdbCommandFailureException as e:
-            raise Exception(f"Unable to download file {remote_path}: {e}")
-
-    def _adb_process_file(self, remote_path, process_routine):
-        """Download a local copy of a file which is only accessible as root.
-        This is a wrapper around process_routine.
-        :param remote_path: Path of the file on the device to process
-        :param process_routine: Function to be called on the local copy of the
-                                downloaded file
-        """
-        # Connect to the device over adb.
-        self._adb_connect()
-        # Check if we have root, if not raise an Exception.
-        self._adb_root_or_die()
-
-        # We create a temporary local file.
-        tmp = tempfile.NamedTemporaryFile()
-        local_path = tmp.name
-        local_name = os.path.basename(tmp.name)
-        new_remote_path = f"/sdcard/Download/{local_name}"
-
-        # We copy the file from the data folder to /sdcard/.
-        cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}")
-        if cp.startswith("cp: ") and "No such file or directory" in cp:
-            raise Exception(f"Unable to process file {remote_path}: File not found")
-        elif cp.startswith("cp: ") and "Permission denied" in cp:
-            raise Exception(f"Unable to process file {remote_path}: Permission denied")
-
-        # We download from /sdcard/ to the local temporary file.
-        self._adb_download(new_remote_path, local_path)
-
-        # Launch the provided process routine!
-        process_routine(local_path)
-
-        # Delete the local copy.
-        tmp.close()
-        # Delete the copy on /sdcard/.
-        self._adb_command(f"rm -f {new_remote_path}")
-        # Disconnect from the device.
-        self._adb_disconnect()
-
-    def run(self):
-        """Run the main procedure.
-        """
-        raise NotImplementedError

mvt/android/modules/adb/chrome_history.py

@@ -1,70 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sqlite3
-import logging
-
-from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-CHROME_HISTORY_PATH = "data/data/com.android.chrome/app_chrome/Default/History"
-
-class ChromeHistory(AndroidExtraction):
-    """This module extracts records from Android's Chrome browsing history."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "visit",
-            "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})"
-        }
-
-    def _parse_db(self, db_path):
-        """Parse a Chrome History database file.
-        :param db_path: Path to the History database to process.
-        """
-        conn = sqlite3.connect(db_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT
-                urls.id,
-                urls.url,
-                visits.id,
-                visits.visit_time,
-                visits.from_visit
-            FROM urls
-            JOIN visits ON visits.url = urls.id
-            ORDER BY visits.visit_time;
-        """)
-
-        for item in cur:
-            self.results.append(dict(
-                id=item[0],
-                url=item[1],
-                visit_id=item[2],
-                timestamp=item[3],
-                isodate=convert_timestamp_to_iso(convert_chrometime_to_unix(item[3])),
-                redirect_source=item[4],
-            ))
-
-        cur.close()
-        conn.close()
-
-        log.info("Extracted a total of %d history items", len(self.results))
-
-    def run(self):
-        self._adb_process_file(os.path.join("/", CHROME_HISTORY_PATH),
-                               self._parse_db)

mvt/android/modules/adb/dumpsys_batterystats.py

@@ -1,45 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import logging
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class DumpsysBatterystats(AndroidExtraction):
-    """This module extracts stats on battery consumption by processes."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._adb_connect()
-
-        stats = self._adb_command("dumpsys batterystats")
-        if self.output_folder:
-            stats_path = os.path.join(self.output_folder,
-                                      "dumpsys_batterystats.txt")
-            with open(stats_path, "w") as handle:
-                handle.write(stats)
-
-            log.info("Records from dumpsys batterystats stored at %s",
-                      stats_path)
-
-        history = self._adb_command("dumpsys batterystats --history")
-        if self.output_folder:
-            history_path = os.path.join(self.output_folder,
-                                        "dumpsys_batterystats_history.txt")
-            with open(history_path, "w") as handle:
-                handle.write(history)
-
-            log.info("History records from dumpsys batterystats stored at %s",
-                     history_path)
-
-        self._adb_disconnect()

mvt/android/modules/adb/dumpsys_packages.py

@@ -1,35 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import logging
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class DumpsysPackages(AndroidExtraction):
-    """This module extracts stats on installed packages."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._adb_connect()
-
-        output = self._adb_command("dumpsys package")
-        if self.output_folder:
-            packages_path = os.path.join(self.output_folder,
-                                         "dumpsys_packages.txt")
-            with open(packages_path, "w") as handle:
-                handle.write(output)
-
-            log.info("Records from dumpsys package stored at %s",
-                     packages_path)
-
-        self._adb_disconnect()

mvt/android/modules/adb/dumpsys_procstats.py

@@ -1,35 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import logging
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class DumpsysProcstats(AndroidExtraction):
-    """This module extracts stats on memory consumption by processes."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._adb_connect()
-
-        output = self._adb_command("dumpsys procstats")
-        if self.output_folder:
-            procstats_path = os.path.join(self.output_folder,
-                                          "dumpsys_procstats.txt")
-            with open(procstats_path, "w") as handle:
-                handle.write(output)
-
-            log.info("Records from dumpsys procstats stored at %s",
-                     procstats_path)
-
-        self._adb_disconnect()

mvt/android/modules/adb/packages.py

@@ -1,111 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import logging
-import pkg_resources
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class Packages(AndroidExtraction):
-    """This module extracts the list of installed packages."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        records = []
-
-        timestamps = [
-            {"event": "package_install", "timestamp": record["timestamp"]},
-            {"event": "package_first_install", "timestamp": record["first_install_time"]},
-            {"event": "package_last_update", "timestamp": record["last_update_time"]},
-        ]
-
-        for ts in timestamps:
-            records.append({
-                "timestamp": ts["timestamp"],
-                "module": self.__class__.__name__,
-                "event": ts["event"],
-                "data": f"{record['package_name']} (system: {record['system']}, third party: {record['third_party']})",
-            })
-
-        return records
-
-    def check_indicators(self):
-        root_packages_path = os.path.join("..", "..", "data", "root_packages.txt")
-        root_packages_string = pkg_resources.resource_string(__name__, root_packages_path)
-        root_packages = root_packages_string.decode("utf-8").split("\n")
-
-        for root_package in root_packages:
-            root_package = root_package.strip()
-            if not root_package:
-                continue
-
-            if root_package in self.results:
-                self.log.warning("Found an installed package related to rooting/jailbreaking: \"%s\"",
-                                 root_package)
-                self.detected.append(root_package)
-
-    def run(self):
-        self._adb_connect()
-
-        packages = self._adb_command("pm list packages -U -u -i -f")
-        for line in packages.split("\n"):
-            line = line.strip()
-            if not line.startswith("package:"):
-                continue
-
-            fields = line.split()
-            file_name, package_name = fields[0].split(":")[1].rsplit("=", 1)
-            installer = fields[1].split("=")[1].strip()
-            if installer == "null":
-                installer = None
-            uid = fields[2].split(":")[1].strip()
-
-            dumpsys = self._adb_command(f"dumpsys package {package_name} | grep -A2 timeStamp").split("\n")
-            timestamp = dumpsys[0].split("=")[1].strip()
-            first_install = dumpsys[1].split("=")[1].strip()
-            last_update = dumpsys[2].split("=")[1].strip()
-
-            self.results.append(dict(
-                package_name=package_name,
-                file_name=file_name,
-                installer=installer,
-                timestamp=timestamp,
-                first_install_time=first_install,
-                last_update_time=last_update,
-                uid=uid,
-                disabled=False,
-                system=False,
-                third_party=False,
-            ))
-
-        cmds = [
-            {"field": "disabled", "arg": "-d"},
-            {"field": "system", "arg": "-s"},
-            {"field": "third_party", "arg": "-3"},
-        ]
-        for cmd in cmds:
-            output = self._adb_command(f"pm list packages {cmd['arg']}")
-            for line in output.split("\n"):
-                line = line.strip()
-                if not line.startswith("package:"):
-                    continue
-
-                package_name = line.split(":", 1)[1]
-                for i, result in enumerate(self.results):
-                    if result["package_name"] == package_name:
-                        self.results[i][cmd["field"]] = True
-
-        self.log.info("Extracted at total of %d installed package names",
-                      len(self.results))
-
-        self._adb_disconnect()

mvt/android/modules/adb/processes.py

@@ -1,54 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import logging
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class Processes(AndroidExtraction):
-    """This module extracts details on running processes."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._adb_connect()
-
-        output = self._adb_command("ps")
-
-        for line in output.split("\n")[1:]:
-            line = line.strip()
-            if line == "":
-                continue
-
-            fields = line.split()
-            proc = dict(
-                user=fields[0],
-                pid=fields[1],
-                parent_pid=fields[2],
-                vsize=fields[3],
-                rss=fields[4],
-            )
-
-            # Sometimes WCHAN is empty, so we need to re-align output fields.
-            if len(fields) == 8:
-                proc["wchan"] = ""
-                proc["pc"] = fields[5]
-                proc["name"] = fields[7]
-            elif len(fields) == 9:
-                proc["wchan"] = fields[5]
-                proc["pc"] = fields[6]
-                proc["name"] = fields[8]
-
-            self.results.append(proc)
-
-        self._adb_disconnect()
-
-        log.info("Extracted records on a total of %d processes", len(self.results))

mvt/android/modules/adb/rootbinaries.py

@@ -1,47 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import logging
-import pkg_resources
-
-from .base import AndroidExtraction
-
-log = logging.getLogger(__name__)
-
-class RootBinaries(AndroidExtraction):
-    """This module extracts the list of installed packages."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        root_binaries_path = os.path.join("..", "..", "data", "root_binaries.txt")
-        root_binaries_string = pkg_resources.resource_string(__name__, root_binaries_path)
-        root_binaries = root_binaries_string.decode("utf-8").split("\n")
-
-        self._adb_connect()
-
-        for root_binary in root_binaries:
-            root_binary = root_binary.strip()
-            if not root_binary:
-                continue
-
-            output = self._adb_command(f"which -a {root_binary}")
-            output = output.strip()
-
-            if not output:
-                continue
-
-            if "which: not found" in output:
-                continue
-
-            self.detected.append(root_binary)
-            self.log.warning("Found root binary \"%s\"", root_binary)
-
-        self._adb_disconnect()

mvt/android/modules/adb/sms.py

@@ -1,91 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sqlite3
-import logging
-
-from .base import AndroidExtraction
-from mvt.common.utils import convert_timestamp_to_iso, check_for_links
-
-log = logging.getLogger(__name__)
-
-SMS_PATH = "data/data/com.google.android.apps.messaging/databases/bugle_db"
-
-class SMS(AndroidExtraction):
-    """This module extracts all SMS messages containing links."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        text = record["text"].replace("\n", "\\n")
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": f"sms_{record['direction']}",
-            "data": f"{record['number']}: \"{text}\""
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for message in self.results:
-            if not "text" in message:
-                continue
-
-            message_links = check_for_links(message["text"])
-            if self.indicators.check_domains(message_links):
-                self.detected.append(message)
-
-    def _parse_db(self, db_path):
-        """Parse an Android bugle_db SMS database file.
-        :param db_path: Path to the Android SMS database file to process
-        """
-        conn = sqlite3.connect(db_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT 
-                ppl.normalized_destination AS number,
-                p.timestamp AS timestamp,
-            CASE WHEN m.sender_id IN 
-            (SELECT _id FROM participants WHERE contact_id=-1)
-            THEN 2 ELSE 1 END incoming, p.text AS text 
-            FROM messages m, conversations c, parts p,
-                 participants ppl, conversation_participants cp
-            WHERE (m.conversation_id = c._id)
-                AND (m._id = p.message_id)
-                AND (cp.conversation_id = c._id)
-                AND (cp.participant_id = ppl._id);
-        """)
-        names = [description[0] for description in cur.description]
-
-        for item in cur:
-            message = dict()
-            for index, value in enumerate(item):
-                message[names[index]] = value
-
-            message["direction"] = ("received" if message["incoming"] == 1 else "sent")
-            message["isodate"] = convert_timestamp_to_iso(message["timestamp"])
-
-            # If we find links in the messages or if they are empty we add
-            # them to the list of results.
-            if check_for_links(message["text"]) or message["text"].strip() == "":
-                self.results.append(message)
-
-        cur.close()
-        conn.close()
-
-        log.info("Extracted a total of %d SMS messages containing links", len(self.results))
-
-    def run(self):
-        try:
-            self._adb_process_file(os.path.join("/", SMS_PATH), self._parse_db)
-        except Exception as e:
-            self.log.error(e)

mvt/android/modules/adb/whatsapp.py

@@ -1,84 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sqlite3
-import logging
-
-from .base import AndroidExtraction
-from mvt.common.utils import convert_timestamp_to_iso, check_for_links
-
-log = logging.getLogger(__name__)
-
-WHATSAPP_PATH = "data/data/com.whatsapp/databases/msgstore.db"
-
-class Whatsapp(AndroidExtraction):
-    """This module extracts all WhatsApp messages containing links."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        text = record["data"].replace("\n", "\\n")
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": f"whatsapp_msg_{record['direction']}",
-            "data": f"\"{text}\""
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for message in self.results:
-            if not "data" in message:
-                continue
-
-            message_links = check_for_links(message["data"])
-            if self.indicators.check_domains(message_links):
-                self.detected.append(message)
-
-    def _parse_db(self, db_path):
-        """Parse an Android msgstore.db WhatsApp database file.
-        :param db_path: Path to the Android WhatsApp database file to process
-        """
-        conn = sqlite3.connect(db_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT * FROM messages;
-        """)
-        names = [description[0] for description in cur.description]
-
-        messages = []
-        for item in cur:
-            message = dict()
-            for index, value in enumerate(item):
-                message[names[index]] = value
-
-            if not message["data"]:
-                continue
-
-            message["direction"] = ("send" if message["key_from_me"] == 1 else "received")
-            message["isodate"] = convert_timestamp_to_iso(message["timestamp"])
-
-            # If we find links in the messages or if they are empty we add them to the list.
-            if check_for_links(message["data"]) or message["data"].strip() == "":
-                messages.append(message)
-
-        cur.close()
-        conn.close()
-
-        log.info("Extracted a total of %d WhatsApp messages containing links", len(messages))
-        self.results = messages
-
-    def run(self):
-        try:
-            self._adb_process_file(os.path.join("/", WHATSAPP_PATH), self._parse_db)
-        except Exception as e:
-            self.log.error(e)

mvt/android/modules/backup/__init__.py

@@ -1,8 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .sms import SMS
-
-BACKUP_MODULES = [SMS,]

mvt/android/modules/backup/sms.py

@@ -1,64 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import json
-import zlib
-
-from mvt.common.module import MVTModule
-from mvt.common.utils import check_for_links
-from mvt.common.utils import convert_timestamp_to_iso
-
-class SMS(MVTModule):
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for message in self.results:
-            if not "body" in message:
-                continue
-
-            message_links = check_for_links(message["body"])
-            if self.indicators.check_domains(message_links):
-                self.detected.append(message)
-
-    def _process_sms_file(self, file_path):
-        self.log.info("Processing SMS backup file at %s", file_path)
-
-        with open(file_path, "rb") as handle:
-            data = zlib.decompress(handle.read())
-            json_data = json.loads(data)
-
-        for entry in json_data:
-            message_links = check_for_links(entry["body"])
-
-            # If we find links in the messages or if they are empty we add them to the list.
-            if message_links or entry["body"].strip() == "":
-                self.results.append(entry)
-
-    def run(self):
-        app_folder = os.path.join(self.base_folder,
-                                  "apps",
-                                  "com.android.providers.telephony",
-                                  "d_f")
-        if not os.path.exists(app_folder):
-            raise FileNotFoundError("Unable to find the SMS backup folder")
-
-        for file_name in os.listdir(app_folder):
-            if not file_name.endswith("_sms_backup"):
-                continue
-
-            file_path = os.path.join(app_folder, file_name)
-            self._process_sms_file(file_path)
-
-        self.log.info("Extracted a total of %d SMS messages containing links",
-                      len(self.results))

mvt/common/__init__.py

@@ -1,4 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE

mvt/common/indicators.py

@@ -1,175 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import json
-
-from .url import URL
-
-class Indicators:
-    """This class is used to parse indicators from a STIX2 file and provide
-    functions to compare extracted artifacts to the indicators.
-    """
-
-    def __init__(self, file_path, log=None):
-        self.file_path = file_path
-        with open(self.file_path, "r") as handle:
-            self.data = json.load(handle)
-
-        self.log = log
-        self.ioc_domains = []
-        self.ioc_processes = []
-        self.ioc_emails = []
-        self.ioc_files = []
-        self._parse_stix_file()
-
-    def _parse_stix_file(self):
-        """Extract IOCs of given type from STIX2 definitions.
-        """
-        for entry in self.data["objects"]:
-            try:
-                if entry["type"] != "indicator":
-                    continue
-            except KeyError:
-                continue
-
-            key, value = entry["pattern"].strip("[]").split("=")
-            value = value.strip("'")
-
-            if key == "domain-name:value":
-                # We force domain names to lower case.
-                value = value.lower()
-                if value not in self.ioc_domains:
-                    self.ioc_domains.append(value)
-            elif key == "process:name":
-                if value not in self.ioc_processes:
-                    self.ioc_processes.append(value)
-            elif key == "email-addr:value":
-                # We force email addresses to lower case.
-                value = value.lower()
-                if value not in self.ioc_emails:
-                    self.ioc_emails.append(value)
-            elif key == "file:name":
-                if value not in self.ioc_files:
-                    self.ioc_files.append(value)
-
-    def check_domain(self, url):
-        # TODO: If the IOC domain contains a subdomain, it is not currently
-        # being matched.
-
-        try:
-            # First we use the provided URL.
-            orig_url = URL(url)
-
-            if orig_url.check_if_shortened():
-                # If it is, we try to retrieve the actual URL making an
-                # HTTP HEAD request.
-                unshortened = orig_url.unshorten()
-
-                # self.log.info("Found a shortened URL %s -> %s",
-                #               url, unshortened)
-
-                # Now we check for any nested URL shorteners.
-                dest_url = URL(unshortened)
-                if dest_url.check_if_shortened():
-                    # self.log.info("Original URL %s appears to shorten another shortened URL %s ... checking!",
-                    #               orig_url.url, dest_url.url)
-                    return self.check_domain(dest_url.url)
-
-                final_url = dest_url
-            else:
-                # If it's not shortened, we just use the original URL object.
-                final_url = orig_url
-        except Exception as e:
-            # If URL parsing failed, we just try to do a simple substring
-            # match.
-            for ioc in self.ioc_domains:
-                if ioc.lower() in url:
-                    self.log.warning("Maybe found a known suspicious domain: %s", url)
-                    return True
-
-            # If nothing matched, we can quit here.
-            return False
-
-        # If all parsing worked, we start walking through available domain indicators.
-        for ioc in self.ioc_domains:
-            # First we check the full domain.
-            if final_url.domain.lower() == ioc:
-                if orig_url.is_shortened and orig_url.url != final_url.url:
-                    self.log.warning("Found a known suspicious domain %s shortened as %s",
-                                     final_url.url, orig_url.url)
-                else:
-                    self.log.warning("Found a known suspicious domain: %s", final_url.url)
-
-                return True
-
-            # Then we just check the top level domain.
-            if final_url.top_level.lower() == ioc:
-                if orig_url.is_shortened and orig_url.url != final_url.url:
-                    self.log.warning("Found a sub-domain matching a suspicious top level %s shortened as %s",
-                                     final_url.url, orig_url.url)
-                else:
-                    self.log.warning("Found a sub-domain matching a suspicious top level: %s", final_url.url)
-
-                return True
-
-    def check_domains(self, urls):
-        """Check the provided list of (suspicious) domains against a list of URLs.
-        :param urls: List of URLs to check
-        """
-        for url in urls:
-            if self.check_domain(url):
-                return True
-
-    def check_process(self, process):
-        """Check the provided process name against the list of process
-        indicators.
-        :param process: Process name to check
-        """
-        if not process:
-            return False
-
-        proc_name = os.path.basename(process)
-        if proc_name in self.ioc_processes:
-            self.log.warning("Found a known suspicious process name \"%s\"", process)
-            return True
-
-        if len(proc_name) == 16:
-            for bad_proc in self.ioc_processes:
-                if bad_proc.startswith(proc_name):
-                    self.log.warning("Found a truncated known suspicious process name \"%s\"", process)
-                    return True
-
-    def check_processes(self, processes):
-        """Check the provided list of processes against the list of
-        process indicators.
-        :param processes: List of processes to check
-        """
-        for process in processes:
-            if self.check_process(process):
-                return True
-
-    def check_email(self, email):
-        """Check the provided email against the list of email indicators.
-        :param email: Suspicious email to check
-        """
-        if not email:
-            return False
-
-        if email.lower() in self.ioc_emails:
-            self.log.warning("Found a known suspicious email address: \"%s\"", email)
-            return True
-
-    def check_file(self, file_path):
-        """Check the provided file path against the list of file indicators.
-        :param file_path: Path or name of the file to check
-        """
-        if not file_path:
-            return False
-
-        file_name = os.path.basename(file_path)
-        if file_name in self.ioc_files:
-            self.log.warning("Found a known suspicious file: \"%s\"", file_path)
-            return True

mvt/common/module.py

@@ -1,172 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import re
-import csv
-import glob
-import logging
-import simplejson as json
-
-from .indicators import Indicators
-
-class MVTModule(object):
-    """This class provides a base for all extraction modules."""
-
-    enabled = True
-    slug = None
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        """Initialize module.
-        :param file_path: Path to the module's database file, if there is any.
-        :param base_folder: Path to the base folder (backup or filesystem dump)
-        :param output_folder: Folder where results will be stored
-        :param fast_mode: Flag to enable or disable slow modules
-        :param log: Handle to logger
-        :param results: Provided list of results entries
-        """
-        self.file_path = file_path
-        self.base_folder = base_folder
-        self.output_folder = output_folder
-        self.fast_mode = fast_mode
-        self.log = log
-        self.indicators = None
-        self.results = results
-        self.detected = []
-        self.timeline = []
-        self.timeline_detected = []
-
-    @classmethod
-    def from_json(cls, json_path, log=None):
-        with open(json_path, "r") as handle:
-            results = json.load(handle)
-            if log:
-                log.info("Loaded %d results from \"%s\"",
-                         len(results), json_path)
-            return cls(results=results, log=log)
-
-    def get_slug(self):
-        if self.slug:
-            return self.slug
-
-        sub = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", self.__class__.__name__)
-        return re.sub("([a-z0-9])([A-Z])", r"\1_\2", sub).lower()
-
-    def _find_paths(self, root_paths):
-        for root_path in root_paths:
-            for found_path in glob.glob(os.path.join(self.base_folder, root_path)):
-                if not os.path.exists(found_path):
-                    continue
-                yield found_path
-
-    def load_indicators(self, file_path):
-        self.indicators = Indicators(file_path, self.log)
-
-    def check_indicators(self):
-        """Check the results of this module againt a provided list of
-        indicators."""
-        raise NotImplementedError
-
-    def save_to_json(self):
-        """Save the collected results to a json file.
-        """
-        if not self.output_folder:
-            return
-
-        name = self.get_slug()
-
-        if self.results:
-            results_file_name = f"{name}.json"
-            results_json_path = os.path.join(self.output_folder, results_file_name)
-            with open(results_json_path, "w") as handle:
-                json.dump(self.results, handle, indent=4)
-
-        if self.detected:
-            detected_file_name = f"{name}_detected.json"
-            detected_json_path = os.path.join(self.output_folder, detected_file_name)
-            with open(detected_json_path, "w") as handle:
-                json.dump(self.detected, handle, indent=4)
-
-    def serialize(self, record):
-        raise NotImplementedError
-
-    def to_timeline(self):
-        """Convert results into a timeline.
-        """
-        for result in self.results:
-            record = self.serialize(result)
-            if type(record) == list:
-                self.timeline.extend(record)
-            else:
-                self.timeline.append(record)
-
-        for detected in self.detected:
-            record = self.serialize(detected)
-            if type(record) == list:
-                self.timeline_detected.extend(record)
-            else:
-                self.timeline_detected.append(record)
-
-        # De-duplicate timeline entries
-        self.timeline = self.timeline_deduplicate(self.timeline)
-        self.timeline_detected = self.timeline_deduplicate(self.timeline_detected)
-
-    def timeline_deduplicate(self, timeline):
-        """Serialize entry as JSON to deduplicate repeated entries"""
-        timeline_set = set()
-        for record in timeline:
-            timeline_set.add(json.dumps(record, sort_keys=True))
-        return [json.loads(record) for record in timeline_set]
-
-    def run(self):
-        """Run the main module procedure.
-        """
-        raise NotImplementedError
-
-
-def run_module(module):
-    module.log.info("Running module %s...", module.__class__.__name__)
-
-    try:
-        module.run()
-    except NotImplementedError:
-        module.log.exception("The run() procedure of module %s was not implemented yet!",
-                             module.__class__.__name__)
-    except FileNotFoundError as e:
-        module.log.error("There might be no data to extract by module %s: %s",
-                         module.__class__.__name__, e)
-    except Exception as e:
-        module.log.exception("Error in running extraction from module %s: %s",
-                             module.__class__.__name__, e)
-    else:
-        try:
-            module.check_indicators()
-        except NotImplementedError:
-            pass
-
-        try:
-            module.to_timeline()
-        except NotImplementedError:
-            pass
-
-        module.save_to_json()
-
-
-def save_timeline(timeline, timeline_path):
-    """Save the timeline in a csv file.
-    :param timeline: List of records to order and store.
-    :param timeline_path: Path to the csv file to store the timeline to.
-    """
-    with open(timeline_path, "a+") as handle:
-        csvoutput = csv.writer(handle, delimiter=",", quotechar="\"")
-        csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"])
-        for event in sorted(timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else ""):
-            csvoutput.writerow([
-                event["timestamp"],
-                event["module"],
-                event["event"],
-                event["data"],
-            ])

mvt/common/options.py

@@ -1,39 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-# From: https://gist.github.com/stanchan/bce1c2d030c76fe9223b5ff6ad0f03db
-
-from click import command, option, Option, UsageError
-
-class MutuallyExclusiveOption(Option):
-    """This class extends click to support mutually exclusive options.
-    """
-
-    def __init__(self, *args, **kwargs):
-        self.mutually_exclusive = set(kwargs.pop("mutually_exclusive", []))
-        help = kwargs.get("help", "")
-        if self.mutually_exclusive:
-            ex_str = ", ".join(self.mutually_exclusive)
-            kwargs["help"] = help + (
-                " NOTE: This argument is mutually exclusive with "
-                "arguments: [" + ex_str + "]."
-            )
-        super(MutuallyExclusiveOption, self).__init__(*args, **kwargs)
-
-    def handle_parse_result(self, ctx, opts, args):
-        if self.mutually_exclusive.intersection(opts) and self.name in opts:
-            raise UsageError(
-                "Illegal usage: `{}` is mutually exclusive with "
-                "arguments `{}`.".format(
-                    self.name,
-                    ", ".join(self.mutually_exclusive)
-                )
-            )
-
-        return super(MutuallyExclusiveOption, self).handle_parse_result(
-            ctx,
-            opts,
-            args
-        )

mvt/common/utils.py

@@ -1,98 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import re
-import datetime
-import hashlib
-
-def convert_mactime_to_unix(timestamp, from_2001=True):
-    """Converts Mac Standard Time to a Unix timestamp.
-    :param timestamp: MacTime timestamp (either int or float)
-    :returns: Unix epoch timestamp
-    """
-    if not timestamp:
-        return None
-
-    # This is to fix formats in case of, for example, SMS messages database
-    # timestamp format.
-    if type(timestamp) == int and len(str(timestamp)) == 18:
-        timestamp = int(str(timestamp)[:9])
-
-    # MacTime counts from 2001-01-01.
-    if from_2001:
-        timestamp = timestamp + 978307200
-
-    # TODO: This is rather ugly. Happens sometimes with invalid timestamps.
-    try:
-        return datetime.datetime.utcfromtimestamp(timestamp)
-    except Exception:
-        return None
-
-
-def convert_chrometime_to_unix(timestamp):
-    """Converts Chrome timestamp to a Unix timestamp.
-    :param timestamp: Chrome timestamp as int
-    :returns: Unix epoch timestamp
-    """
-    epoch_start = datetime.datetime(1601, 1 , 1)
-    delta = datetime.timedelta(microseconds=timestamp)
-    return epoch_start + delta
-
-
-def convert_timestamp_to_iso(timestamp):
-    """Converts Unix timestamp to ISO string.
-    :param timestamp: Unix timestamp
-    :returns: ISO timestamp string in YYYY-mm-dd HH:MM:SS.ms format
-    """
-    try:
-        return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
-    except Exception:
-        return None
-
-def check_for_links(text):
-    """Checks if a given text contains HTTP links.
-    :param text: Any provided text
-    :returns: Search results
-    """
-    return re.findall("(?P<url>https?://[^\s]+)", text, re.IGNORECASE)
-
-def get_sha256_from_file_path(file_path):
-    """Calculate the SHA256 hash of a file from a file path.
-    :param file_path: Path to the file to hash
-    :returns: The SHA256 hash string
-    """
-    sha256_hash = hashlib.sha256()
-    with open(file_path, "rb") as handle:
-        for byte_block in iter(lambda: handle.read(4096), b""):
-            sha256_hash.update(byte_block)
-
-    return sha256_hash.hexdigest()
-
-# Note: taken from here:
-# https://stackoverflow.com/questions/57014259/json-dumps-on-dictionary-with-bytes-for-keys
-def keys_bytes_to_string(obj):
-    """Convert object keys from bytes to string.
-    :param obj: Object to convert from bytes to string.
-    :returns: Converted object.
-    """
-    new_obj = {}
-    if not isinstance(obj, dict):
-        if isinstance(obj, (tuple, list, set)):
-            value = [keys_bytes_to_string(x) for x in obj]
-            return value
-        else:
-            return obj
-
-    for key, value in obj.items():
-        if isinstance(key, bytes):
-            key = key.decode()
-        if isinstance(value, dict):
-            value = keys_bytes_to_string(value)
-        elif isinstance(value, (tuple, list, set)):
-            value = [keys_bytes_to_string(x) for x in value]
-        new_obj[key] = value
-
-    return new_obj

mvt/ios/__init__.py

@@ -1,6 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .cli import cli

mvt/ios/cli.py

@@ -1,215 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sys
-import click
-import tarfile
-import logging
-from rich.logging import RichHandler
-
-from mvt.common.module import run_module, save_timeline
-from mvt.common.options import MutuallyExclusiveOption
-from mvt.common.indicators import Indicators
-
-from .decrypt import DecryptBackup
-from .modules.fs import BACKUP_MODULES, FS_MODULES
-
-# Setup logging using Rich.
-LOG_FORMAT = "[%(name)s] %(message)s"
-logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
-    RichHandler(show_path=False, log_time_format="%X")])
-log = logging.getLogger(__name__)
-
-# Help messages of repeating options.
-OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results"
-
-
-#==============================================================================
-# Main
-#==============================================================================
-@click.group(invoke_without_command=False)
-def cli():
-    return
-
-
-#==============================================================================
-# Command: decrypt-backup
-#==============================================================================
-@cli.command("decrypt-backup", help="Decrypt an encrypted iTunes backup")
-@click.option("--destination", "-d", required=True,
-              help="Path to the folder where to store the decrypted backup")
-@click.option("--password", "-p", cls=MutuallyExclusiveOption,
-              help="Password to use to decrypt the backup",
-              mutually_exclusive=["key_file"])
-@click.option("--key-file", "-k", cls=MutuallyExclusiveOption,
-              type=click.Path(exists=True),
-              help="File containing raw encryption key to use to decrypt the backup",
-              mutually_exclusive=["password"])
-@click.argument("BACKUP_PATH", type=click.Path(exists=True))
-def decrypt_backup(destination, password, key_file, backup_path):
-    backup = DecryptBackup(backup_path, destination)
-    if password:
-        backup.decrypt_with_password(password)
-    elif key_file:
-        backup.decrypt_with_key_file(key_file)
-    else:
-        raise click.ClickException("Missing required option. Specify either "
-                                   "--password or --key-file.")
-
-
-#==============================================================================
-# Command: check-backup
-#==============================================================================
-@cli.command("check-backup", help="Extract artifacts from an iTunes backup")
-@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE)
-@click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
-@click.argument("BACKUP_PATH", type=click.Path(exists=True))
-def check_backup(iocs, output, fast, backup_path, list_modules, module):
-    if list_modules:
-        log.info("Following is the list of available check-backup modules:")
-        for backup_module in BACKUP_MODULES:
-            log.info(" - %s", backup_module.__name__)
-
-        return
-
-    log.info("Checking iTunes backup located at: %s", backup_path)
-
-    if iocs:
-        # Pre-load indicators for performance reasons.
-        log.info("Loading indicators from provided file at: %s", iocs)
-        indicators = Indicators(iocs)
-
-    timeline = []
-    timeline_detected = []
-    for backup_module in BACKUP_MODULES:
-        if module and backup_module.__name__ != module:
-            continue
-
-        m = backup_module(base_folder=backup_path, output_folder=output, fast_mode=fast,
-                          log=logging.getLogger(backup_module.__module__))
-        m.is_backup = True
-
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
-        run_module(m)
-        timeline.extend(m.timeline)
-        timeline_detected.extend(m.timeline_detected)
-
-    if output:
-        if len(timeline) > 0:
-            save_timeline(timeline, os.path.join(output, "timeline.csv"))
-        if len(timeline_detected) > 0:
-            save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv"))
-
-
-#==============================================================================
-# Command: check-fs
-#==============================================================================
-@cli.command("check-fs", help="Extract artifacts from a full filesystem dump")
-@click.option("--iocs", "-i", type=click.Path(exists=True), help="Path to indicators file")
-@click.option("--output", "-o", type=click.Path(exists=True), help=OUTPUT_HELP_MESSAGE)
-@click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
-@click.argument("DUMP_PATH", type=click.Path(exists=True))
-def check_fs(iocs, output, fast, dump_path, list_modules, module):
-    if list_modules:
-        log.info("Following is the list of available check-fs modules:")
-        for fs_module in FS_MODULES:
-            log.info(" - %s", fs_module.__name__)
-
-        return
-
-    log.info("Checking filesystem dump located at: %s", dump_path)
-
-    if iocs:
-        # Pre-load indicators for performance reasons.
-        log.info("Loading indicators from provided file at: %s", iocs)
-        indicators = Indicators(iocs)
-
-    timeline = []
-    timeline_detected = []
-    for fs_module in FS_MODULES:
-        if module and fs_module.__name__ != module:
-            continue
-
-        m = fs_module(base_folder=dump_path, output_folder=output, fast_mode=fast,
-                      log=logging.getLogger(fs_module.__module__))
-
-        m.is_fs_dump = True
-
-        if iocs:
-            indicators.log = m.log
-            m.indicators = indicators
-
-        run_module(m)
-        timeline.extend(m.timeline)
-        timeline_detected.extend(m.timeline_detected)
-
-    if output:
-        if len(timeline) > 0:
-            save_timeline(timeline, os.path.join(output, "timeline.csv"))
-        if len(timeline_detected) > 0:
-            save_timeline(timeline_detected, os.path.join(output, "timeline_detected.csv"))
-
-
-#==============================================================================
-# Command: check-iocs
-#==============================================================================
-@cli.command("check-iocs", help="Compare stored JSON results to provided indicators")
-@click.option("--iocs", "-i", required=True, type=click.Path(exists=True),
-              help="Path to indicators file")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
-@click.argument("FOLDER", type=click.Path(exists=True))
-def check_iocs(iocs, list_modules, module, folder):
-    all_modules = []
-    for entry in BACKUP_MODULES + FS_MODULES + SYSDIAGNOSE_MODULES:
-        if entry not in all_modules:
-            all_modules.append(entry)
-
-    if list_modules:
-        log.info("Following is the list of available check-iocs modules:")
-        for iocs_module in all_modules:
-            log.info(" - %s", iocs_module.__name__)
-
-        return
-
-    log.info("Checking stored results against provided indicators...")
-
-    # Pre-load indicators for performance reasons.
-    log.info("Loading indicators from provided file at: %s", iocs)
-    indicators = Indicators(iocs)
-
-    for file_name in os.listdir(folder):
-        name_only, ext = os.path.splitext(file_name)
-        file_path = os.path.join(folder, file_name)
-
-        for iocs_module in all_modules:
-            if module and iocs_module.__name__ != module:
-                continue
-
-            if iocs_module().get_slug() != name_only:
-                continue
-
-            log.info("Loading results from \"%s\" with module %s", file_name,
-                     iocs_module.__name__)
-
-            m = iocs_module.from_json(file_path,
-                                      log=logging.getLogger(iocs_module.__module__))
-
-            indicators.log = m.log
-            m.indicators = indicators
-
-            try:
-                m.check_indicators()
-            except NotImplementedError:
-                continue

mvt/ios/decrypt.py

@@ -1,113 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import shutil
-import sqlite3
-import logging
-import binascii
-from iOSbackup import iOSbackup
-
-log = logging.getLogger(__name__)
-
-class DecryptBackup:
-    """This class provides functions to decrypt an encrypted iTunes backup
-    using either a password or a key file.
-    """
-
-    def __init__(self, backup_path, dest_path):
-        """Decrypts an encrypted iOS backup.
-        :param backup_path: Path to the encrypted backup folder
-        :param dest_path: Path to the folder where to store the decrypted backup
-        """
-        self.backup_path = backup_path
-        self.dest_path = dest_path
-        self._backup = None
-
-    def _process_backup(self):
-        manifest_path = os.path.join(self.dest_path, "Manifest.db")
-        # We extract a decrypted Manifest.db.
-        self._backup.getManifestDB()
-        # We store it to the destination folder.
-        shutil.copy(self._backup.manifestDB, manifest_path)
-
-        for item in self._backup.getBackupFilesList():
-            try:
-                file_id = item["backupFile"]
-                relative_path = item["relativePath"]
-                domain = item["domain"]
-
-                # This may be a partial backup. Skip files from the manifest which do not exist locally.
-                source_file_path = os.path.join(self.backup_path, file_id[0:2], file_id)
-                if not os.path.exists(source_file_path):
-                    log.debug("Skipping file %s. File not found in encrypted backup directory.",
-                              source_file_path)
-                    continue
-
-                item_folder = os.path.join(self.dest_path, file_id[0:2])
-                if not os.path.exists(item_folder):
-                    os.makedirs(item_folder)
-
-                # iOSBackup getFileDecryptedCopy() claims to read a "file" parameter
-                # but the code actually is reading the "manifest" key.
-                # Add manifest plist to both keys to handle this.
-                item["manifest"] = item["file"]
-
-                self._backup.getFileDecryptedCopy(manifestEntry=item,
-                                                  targetName=file_id,
-                                                  targetFolder=item_folder)
-                log.info("Decrypted file %s [%s] to %s/%s", relative_path, domain, item_folder, file_id)
-            except Exception as e:
-                log.error("Failed to decrypt file %s: %s", relative_path, e)
-
-    def decrypt_with_password(self, password):
-        """Decrypts an encrypted iOS backup.
-        :param password: Password to use to decrypt the original backup
-        """
-        log.info("Decrypting iOS backup at path %s with password", self.backup_path)
-
-        if not os.path.exists(self.dest_path):
-            os.makedirs(self.dest_path)
-
-        try:
-            self._backup = iOSbackup(udid=os.path.basename(self.backup_path),
-                                     cleartextpassword=password,
-                                     backuproot=os.path.dirname(self.backup_path))
-        except Exception as e:
-            log.exception(e)
-            log.critical("Failed to decrypt backup. Did you provide the correct password?")
-            return
-        else:
-            self._process_backup()
-
-    def decrypt_with_key_file(self, key_file):
-        """Decrypts an encrypted iOS backup using a key file.
-        :param key_file: File to read the key bytes to decrypt the backup
-        """
-        log.info("Decrypting iOS backup at path %s with key file %s",
-                 self.backup_path, key_file)
-
-        if not os.path.exists(self.dest_path):
-            os.makedirs(self.dest_path)
-
-        with open(key_file, "rb") as handle:
-            key_bytes = handle.read()
-
-        # Key should be 64 hex encoded characters (32 raw bytes)
-        if len(key_bytes) != 64:
-            log.critical("Invalid key from key file. Did you provide the correct key file?")
-            return
-
-        try:
-            key_bytes_raw = binascii.unhexlify(key_bytes)
-            self._backup = iOSbackup(udid=os.path.basename(self.backup_path),
-                                     derivedkey=key_bytes_raw,
-                                     backuproot=os.path.dirname(self.backup_path))
-        except Exception as e:
-            log.exception(e)
-            log.critical("Failed to decrypt backup. Did you provide the correct key file?")
-            return
-        else:
-            self._process_backup()

mvt/ios/modules/__init__.py

@@ -1,4 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE

mvt/ios/modules/fs/__init__.py

@@ -1,42 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .manifest import Manifest
-from .contacts import Contacts
-from .net_netusage import Netusage
-from .net_datausage import Datausage
-from .safari_history import SafariHistory
-from .safari_favicon import SafariFavicon
-from .safari_browserstate import SafariBrowserState
-from .webkit_indexeddb import WebkitIndexedDB
-from .webkit_localstorage import WebkitLocalStorage
-from .webkit_safariviewservice import WebkitSafariViewService
-from .webkit_session_resource_log import WebkitSessionResourceLog
-from .chrome_history import ChromeHistory
-from .chrome_favicon import ChromeFavicon
-from .firefox_history import FirefoxHistory
-from .firefox_favicon import FirefoxFavicon
-from .version_history import IOSVersionHistory
-from .idstatuscache import IDStatusCache
-from .locationd import LocationdClients
-from .interactionc import InteractionC
-from .sms import SMS
-from .sms_attachments import SMSAttachments
-from .calls import Calls
-from .whatsapp import Whatsapp
-from .cache_files import CacheFiles
-from .filesystem import Filesystem
-
-BACKUP_MODULES = [SafariBrowserState, SafariHistory, Datausage, SMS, SMSAttachments,
-                  ChromeHistory, ChromeFavicon, WebkitSessionResourceLog,
-                  Calls, IDStatusCache, LocationdClients, InteractionC,
-                  FirefoxHistory, FirefoxFavicon, Contacts, Manifest, Whatsapp]
-
-FS_MODULES = [IOSVersionHistory, SafariHistory, SafariFavicon, SafariBrowserState,
-              WebkitIndexedDB, WebkitLocalStorage, WebkitSafariViewService,
-              WebkitSessionResourceLog, Datausage, Netusage, ChromeHistory,
-              ChromeFavicon, Calls, IDStatusCache, SMS, SMSAttachments,
-              LocationdClients, InteractionC, FirefoxHistory, FirefoxFavicon,
-              Contacts, CacheFiles, Whatsapp, Filesystem]

mvt/ios/modules/fs/base.py

@@ -1,55 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import glob
-
-from mvt.common.module import MVTModule
-
-class IOSExtraction(MVTModule):
-    """This class provides a base for all iOS filesystem/backup extraction modules."""
-
-    is_backup = False
-    is_fs_dump = False
-    is_sysdiagnose = False
-
-    def _find_ios_database(self, backup_ids=None, root_paths=[]):
-        """Try to locate the module's database file from either an iTunes
-        backup or a full filesystem dump.
-        :param backup_id: iTunes backup database file's ID (or hash).
-        """
-        file_path = None
-        # First we check if the was an explicit file path specified.
-        if not self.file_path:
-            # If not, we first try with backups.
-            # We construct the path to the file according to the iTunes backup
-            # folder structure, if we have a valid ID.
-            if backup_ids:
-                for backup_id in backup_ids:
-                    file_path = os.path.join(self.base_folder, backup_id[0:2], backup_id)
-                    # If we found the correct backup file, then we stop searching.
-                    if os.path.exists(file_path):
-                        break
-
-            # If this file does not exist we might be processing a full
-            # filesystem dump (checkra1n all the things!).
-            if not file_path or not os.path.exists(file_path):
-                # We reset the file_path.
-                file_path = None
-                for root_path in root_paths:
-                    for found_path in glob.glob(os.path.join(self.base_folder, root_path)):
-                        # If we find a valid path, we set file_path.
-                        if os.path.exists(found_path):
-                            file_path = found_path
-                            break
-
-                        # Otherwise, we reset the file_path again.
-                        file_path = None
-
-        # If we do not find any, we fail.
-        if file_path:
-            self.file_path = file_path
-        else:
-            raise FileNotFoundError("Unable to find the module's database file")

mvt/ios/modules/fs/cache_files.py

@@ -1,77 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import sqlite3
-
-from .base import IOSExtraction
-
-class CacheFiles(IOSExtraction):
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        records = []
-        for item in self.results[record]:
-            records.append({
-                "timestamp": item["isodate"],
-                "module": self.__class__.__name__,
-                "event": "cache_response",
-                "data": f"{record} recorded visit to URL {item['url']}"
-            })
-
-        return records
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        self.detected = {}
-        for key, items in self.results.items():
-            for item in items:
-                if self.indicators.check_domain(item["url"]):
-                    if key not in self.detected:
-                        self.detected[key] = [item,]
-                    else:
-                        self.detected[key].append(item)
-
-    def _process_cache_file(self, file_path):
-        self.log.info("Processing cache file at path: %s", file_path)
-
-        conn = sqlite3.connect(file_path)
-        cur = conn.cursor()
-
-        try:
-            cur.execute("SELECT * FROM cfurl_cache_response;")
-        except sqlite3.OperationalError:
-            return
-
-        key_name = os.path.relpath(file_path, self.base_folder)
-        if not key_name in self.results:
-            self.results[key_name] = []
-
-        for row in cur:
-            self.results[key_name].append(dict(
-                entry_id=row[0],
-                version=row[1],
-                hash_value=row[2],
-                storage_policy=row[3],
-                url=row[4],
-                isodate=row[5],
-            ))
-
-    def run(self):
-        self.results = {}
-        for root, dirs, files in os.walk(self.base_folder):
-            for file_name in files:
-                if file_name != "Cache.db":
-                    continue
-
-                file_path = os.path.join(root, file_name)
-                self._process_cache_file(file_path)

mvt/ios/modules/fs/calls.py

@@ -1,60 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-from .base import IOSExtraction
-
-CALLS_BACKUP_IDS = [
-    "5a4935c78a5255723f707230a451d79c540d2741",
-]
-CALLS_ROOT_PATHS = [
-    "private/var/mobile/Library/CallHistoryDB/CallHistory.storedata"
-]
-
-class Calls(IOSExtraction):
-    """This module extracts phone calls details"""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "call",
-            "data": f"From {record['number']} using {record['provider']} during {record['duration']} seconds"
-        }
-
-    def run(self):
-        self._find_ios_database(backup_ids=CALLS_BACKUP_IDS, root_paths=CALLS_ROOT_PATHS)
-        self.log.info("Found Calls database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT
-                ZDATE, ZDURATION, ZLOCATION, ZADDRESS, ZSERVICE_PROVIDER
-                FROM ZCALLRECORD;
-        """)
-        names = [description[0] for description in cur.description]
-
-        for entry in cur:
-            self.results.append({
-                "isodate": convert_timestamp_to_iso(convert_mactime_to_unix(entry[0])),
-                "duration": entry[1],
-                "location": entry[2],
-                "number": entry[3].decode("utf-8") if entry[3] and entry[3] is bytes else entry[3],
-                "provider": entry[4]
-            })
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d calls", len(self.results))

mvt/ios/modules/fs/chrome_favicon.py

@@ -1,78 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-CHROME_FAVICON_BACKUP_IDS = [
-    "55680ab883d0fdcffd94f959b1632e5fbbb18c5b"
-]
-
-# TODO: Confirm Chrome database path.
-CHROME_FAVICON_ROOT_PATHS = [
-    "private/var/mobile/Containers/Data/Application/*/Library/Application Support/Google/Chrome/Default/Favicons",
-]
-
-class ChromeFavicon(IOSExtraction):
-    """This module extracts all Chrome favicon records."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "new_favicon",
-            "data": f"{record['icon_url']} from {record['url']}"
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["icon_url"]):
-                self.detected.append(result)
-
-    def run(self):
-        self._find_ios_database(backup_ids=CHROME_FAVICON_BACKUP_IDS, root_paths=CHROME_FAVICON_ROOT_PATHS)
-        self.log.info("Found Chrome favicon cache database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-
-        # Fetch icon cache
-        cur = conn.cursor()
-        cur.execute("""SELECT
-                icon_mapping.page_url,
-                favicons.url,
-                favicon_bitmaps.last_updated,
-                favicon_bitmaps.last_requested
-            FROM icon_mapping
-            JOIN favicon_bitmaps ON icon_mapping.icon_id = favicon_bitmaps.icon_id
-            JOIN favicons ON icon_mapping.icon_id = favicons.id
-            ORDER BY icon_mapping.id;""")
-
-        items = []
-        for item in cur:
-            last_timestamp = int(item[2]) or int(item[3])
-            items.append(dict(
-                url=item[0],
-                icon_url=item[1],
-                timestamp=last_timestamp,
-                isodate=convert_timestamp_to_iso(convert_chrometime_to_unix(last_timestamp)),
-            ))
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d favicon records", len(items))
-        self.results = sorted(items, key=lambda item: item["isodate"])

mvt/ios/modules/fs/chrome_history.py

@@ -1,69 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from mvt.common.utils import convert_chrometime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-CHROME_HISTORY_BACKUP_IDS = [
-    "faf971ce92c3ac508c018dce1bef2a8b8e9838f1",
-]
-
-# TODO: Confirm Chrome database path.
-CHROME_HISTORY_ROOT_PATHS = [
-    "private/var/mobile/Containers/Data/Application/*/Library/Application Support/Google/Chrome/Default/History",
-]
-
-class ChromeHistory(IOSExtraction):
-    """This module extracts all Chome visits."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "visit",
-            "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})"
-        }
-
-    def run(self):
-        self._find_ios_database(backup_ids=CHROME_HISTORY_BACKUP_IDS, root_paths=CHROME_HISTORY_ROOT_PATHS)
-        self.log.info("Found Chrome history database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT
-                urls.id,
-                urls.url,
-                visits.id,
-                visits.visit_time,
-                visits.from_visit
-            FROM urls
-            JOIN visits ON visits.url = urls.id
-            ORDER BY visits.visit_time;
-        """)
-
-        for item in cur:
-            self.results.append(dict(
-                id=item[0],
-                url=item[1],
-                visit_id=item[2],
-                timestamp=item[3],
-                isodate=convert_timestamp_to_iso(convert_chrometime_to_unix(item[3])),
-                redirect_source=item[4],
-            ))
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d history items", len(self.results))

mvt/ios/modules/fs/contacts.py

@@ -1,52 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from .base import IOSExtraction
-
-CONTACTS_BACKUP_IDS = [
-    "31bb7ba8914766d4ba40d6dfb6113c8b614be442",
-]
-CONTACTS_ROOT_PATHS = [
-    "private/var/mobile/Library/AddressBook/AddressBook.sqlitedb",
-]
-
-class Contacts(IOSExtraction):
-    """This module extracts all contact details from the phone's address book."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._find_ios_database(backup_ids=CONTACTS_BACKUP_IDS, root_paths=CONTACTS_ROOT_PATHS)
-        self.log.info("Found Contacts database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT
-                multi.value, person.first, person.middle, person.last,
-                person.organization
-            FROM ABPerson person, ABMultiValue multi
-            WHERE person.rowid = multi.record_id and multi.value not null
-            ORDER by person.rowid ASC;
-        """)
-        names = [description[0] for description in cur.description]
-
-        for entry in cur:
-            new_contact = dict()
-            for index, value in enumerate(entry):
-                new_contact[names[index]] = value
-
-            self.results.append(new_contact)
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d contacts from the address book", len(self.results))

mvt/ios/modules/fs/filesystem.py

@@ -1,51 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import datetime
-
-from mvt.common.utils import convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-class Filesystem(IOSExtraction):
-    """This module extracts creation and modification date of files from a
-    full file-system dump."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["modified"],
-            "module": self.__class__.__name__,
-            "event": f"file_modified",
-            "data": record["file_path"],
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if self.indicators.check_file(result["file_path"]):
-                self.detected.append(result)
-
-    def run(self):
-        for root, dirs, files in os.walk(self.base_folder):
-            for file_name in files:
-                try:
-                    file_path = os.path.join(root, file_name)
-                    result = {
-                        "file_path": os.path.relpath(file_path, self.base_folder),
-                        "modified": convert_timestamp_to_iso(datetime.datetime.utcfromtimestamp(os.stat(file_path).st_mtime)),
-                    }
-                except:
-                    continue
-                else:
-                    self.results.append(result)

mvt/ios/modules/fs/firefox_favicon.py

@@ -1,82 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from datetime import datetime
-from mvt.common.url import URL
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-FIREFOX_HISTORY_BACKUP_IDS = [
-    "2e57c396a35b0d1bcbc624725002d98bd61d142b",
-]
-FIREFOX_HISTORY_ROOT_PATHS = [
-    "private/var/mobile/profile.profile/browser.db",
-]
-
-class FirefoxFavicon(IOSExtraction):
-    """This module extracts all Firefox favicon"""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "firefox_history",
-            "data": f"Firefox favicon {record['url']} when visiting {record['history_url']}",
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["history_url"]):
-                self.detected.append(result)
-
-    def run(self):
-        self._find_ios_database(backup_ids=FIREFOX_HISTORY_BACKUP_IDS, root_paths=FIREFOX_HISTORY_ROOT_PATHS)
-        self.log.info("Found Firefox favicon database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT
-                favicons.id,
-                favicons.url,
-                favicons.width,
-                favicons.height,
-                favicons.type,
-                favicons.date,
-                history.id,
-                history.url
-            FROM favicons
-            INNER JOIN favicon_sites ON favicon_sites.faviconID = favicons.id
-            INNER JOIN history ON favicon_sites.siteID = history.id;
-        """)
-
-        for item in cur:
-            self.results.append(dict(
-                id=item[0],
-                url=item[1],
-                width=item[2],
-                height=item[3],
-                type=item[4],
-                isodate=convert_timestamp_to_iso(datetime.utcfromtimestamp(item[5])),
-                history_id=item[6],
-                history_url=item[7]
-            ))
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d history items", len(self.results))

mvt/ios/modules/fs/firefox_history.py

@@ -1,78 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from datetime import datetime
-from mvt.common.url import URL
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-FIREFOX_HISTORY_BACKUP_IDS = [
-    "2e57c396a35b0d1bcbc624725002d98bd61d142b",
-]
-FIREFOX_HISTORY_ROOT_PATHS = [
-    "private/var/mobile/profile.profile/browser.db",
-]
-
-class FirefoxHistory(IOSExtraction):
-    """This module extracts all Firefox visits and tries to detect potential
-    network injection attacks."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "firefox_history",
-            "data": f"Firefox visit with ID {record['id']} to URL: {record['url']}",
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if self.indicators.check_domain(result["url"]):
-                self.detected.append(result)
-
-    def run(self):
-        self._find_ios_database(backup_ids=FIREFOX_HISTORY_BACKUP_IDS, root_paths=FIREFOX_HISTORY_ROOT_PATHS)
-        self.log.info("Found Firefox history database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-        cur.execute("""
-            SELECT
-                visits.id,
-                visits.date/1000000,
-                history.url,
-                history.title,
-                visits.is_local,
-                visits.type
-            FROM visits, history
-            WHERE visits.siteID = history.id;
-        """)
-
-        for item in cur:
-            self.results.append(dict(
-                id=item[0],
-                isodate=convert_timestamp_to_iso(datetime.utcfromtimestamp(item[1])),
-                url=item[2],
-                title=item[3],
-                i1000000s_local=item[4],
-                type=item[5]
-            ))
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d history items", len(self.results))

mvt/ios/modules/fs/idstatuscache.py

@@ -1,86 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import glob
-import biplist
-import collections
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-IDSTATUSCACHE_BACKUP_IDS = [
-    "6b97989189901ceaa4e5be9b7f05fb584120e27b",
-]
-IDSTATUSCACHE_ROOT_PATHS = [
-    "private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist",
-]
-
-class IDStatusCache(IOSExtraction):
-    """Extracts Apple Authentication information from idstatuscache.plist"""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "lookup",
-            "data": f"Lookup of {record['user']} within {record['package']} (Status {record['idstatus']})"
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if result["user"].startswith("mailto:"):
-                email = result["user"][7:].strip("'")
-                if self.indicators.check_email(email):
-                    self.detected.append(result)
-                    continue
-
-            if "\\x00\\x00" in result["user"]:
-                self.log.warning("Found an ID Status Cache entry with suspicious patterns: %s",
-                                 result["user"])
-                self.detected.append(result)
-
-    def run(self):
-        self._find_ios_database(backup_ids=IDSTATUSCACHE_BACKUP_IDS, root_paths=IDSTATUSCACHE_ROOT_PATHS)
-        self.log.info("Found IDStatusCache plist at path: %s", self.file_path)
-
-        file_plist = biplist.readPlist(self.file_path)
-
-        id_status_cache_entries = []
-        for app in file_plist:
-            if not isinstance(file_plist[app], dict):
-                continue
-
-            for entry in file_plist[app]:
-                try:
-                    lookup_date = file_plist[app][entry]["LookupDate"]
-                    id_status = file_plist[app][entry]["IDStatus"]
-                except KeyError:
-                    continue
-
-                id_status_cache_entries.append({
-                    "package": app,
-                    "user": entry.replace("\x00", "\\x00"),
-                    "isodate": convert_timestamp_to_iso(convert_mactime_to_unix(lookup_date)),
-                    "idstatus": id_status,
-                })
-
-        entry_counter = collections.Counter([entry["user"] for entry in id_status_cache_entries])
-        for entry in id_status_cache_entries:
-            # Add total count of occurrences to the status cache entry
-            entry["occurrences"] = entry_counter[entry["user"]]
-            self.results.append(entry)
-
-        self.log.info("Extracted a total of %d ID Status Cache entries", len(self.results))

mvt/ios/modules/fs/interactionc.py

@@ -1,174 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-from base64 import b64encode
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-INTERACTIONC_BACKUP_IDS = [
-    "1f5a521220a3ad80ebfdc196978df8e7a2e49dee",
-]
-INTERACTIONC_ROOT_PATHS = [
-    "private/var/mobile/Library/CoreDuet/People/interactionC.db",
-]
-
-class InteractionC(IOSExtraction):
-    """This module extracts data from InteractionC db."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-        self.timestamps = [
-            "start_date",
-            "end_date",
-            "interactions_creation_date",
-            "contacts_creation_date",
-            "first_incoming_recipient_date",
-            "first_incoming_sender_date",
-            "first_outgoing_recipient_date",
-            "last_incoming_sender_date",
-            "last_incoming_recipient_date",
-            "last_outgoing_recipient_date",
-        ]
-
-    def serialize(self, record):
-        records = []
-        processed = []
-        for ts in self.timestamps:
-            # Check if the record has the current timestamp.
-            if ts not in record or not record[ts]:
-                continue
-
-            # Check if the timestamp was already processed.
-            if record[ts] in processed:
-                continue
-
-            records.append({
-                "timestamp": record[ts],
-                "module": self.__class__.__name__,
-                "event": ts,
-                "data": f"[{record['bundle_id']}] {record['account']} - from {record['sender_display_name']} " \
-                        f"({record['sender_identifier']}) to {record['recipient_display_name']} " \
-                        f"({record['recipient_identifier']}): {record['content']}"
-            })
-            processed.append(record[ts])
-
-        return records
-
-    def run(self):
-        self._find_ios_database(backup_ids=INTERACTIONC_BACKUP_IDS, root_paths=INTERACTIONC_ROOT_PATHS)
-        self.log.info("Found InteractionC database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-
-        # TODO: Support all versions.
-        # Taken from:
-        # https://github.com/mac4n6/APOLLO/blob/master/modules/interaction_contact_interactions.txt
-        cur.execute("""
-            SELECT
-                ZINTERACTIONS.ZSTARTDATE,
-                ZINTERACTIONS.ZENDDATE,
-                ZINTERACTIONS.ZBUNDLEID,
-                ZINTERACTIONS.ZACCOUNT,
-                ZINTERACTIONS.ZTARGETBUNDLEID,
-                CASE ZINTERACTIONS.ZDIRECTION
-                    WHEN '0' THEN 'INCOMING'
-                    WHEN '1' THEN 'OUTGOING'
-                END 'DIRECTION',
-                ZCONTACTS.ZDISPLAYNAME,
-                ZCONTACTS.ZIDENTIFIER,
-                ZCONTACTS.ZPERSONID,
-                RECEIPIENTCONACT.ZDISPLAYNAME,
-                RECEIPIENTCONACT.ZIDENTIFIER,
-                RECEIPIENTCONACT.ZPERSONID,
-                ZINTERACTIONS.ZRECIPIENTCOUNT,
-                ZINTERACTIONS.ZDOMAINIDENTIFIER,
-                ZINTERACTIONS.ZISRESPONSE,
-                ZATTACHMENT.ZCONTENTTEXT,
-                ZATTACHMENT.ZUTI,
-                ZATTACHMENT.ZCONTENTURL,
-                ZATTACHMENT.ZSIZEINBYTES,
-                ZATTACHMENT.ZPHOTOLOCALIDENTIFIER,
-                HEX(ZATTACHMENT.ZIDENTIFIER),
-                ZATTACHMENT.ZCLOUDIDENTIFIER,
-                ZCONTACTS.ZINCOMINGRECIPIENTCOUNT,
-                ZCONTACTS.ZINCOMINGSENDERCOUNT,
-                ZCONTACTS.ZOUTGOINGRECIPIENTCOUNT,
-                ZINTERACTIONS.ZCREATIONDATE,
-                ZCONTACTS.ZCREATIONDATE,
-                ZCONTACTS.ZFIRSTINCOMINGRECIPIENTDATE,
-                ZCONTACTS.ZFIRSTINCOMINGSENDERDATE,
-                ZCONTACTS.ZFIRSTOUTGOINGRECIPIENTDATE,
-                ZCONTACTS.ZLASTINCOMINGSENDERDATE,
-                ZCONTACTS.ZLASTINCOMINGRECIPIENTDATE,
-                ZCONTACTS.ZLASTOUTGOINGRECIPIENTDATE,
-                ZCONTACTS.ZCUSTOMIDENTIFIER,
-                ZINTERACTIONS.ZCONTENTURL,
-                ZINTERACTIONS.ZLOCATIONUUID,
-                ZINTERACTIONS.ZGROUPNAME,
-                ZINTERACTIONS.ZDERIVEDINTENTIDENTIFIER,
-                ZINTERACTIONS.Z_PK
-        FROM ZINTERACTIONS
-            LEFT JOIN ZCONTACTS ON ZINTERACTIONS.ZSENDER = ZCONTACTS.Z_PK
-            LEFT JOIN Z_1INTERACTIONS ON ZINTERACTIONS.Z_PK == Z_1INTERACTIONS.Z_3INTERACTIONS
-            LEFT JOIN ZATTACHMENT ON Z_1INTERACTIONS.Z_1ATTACHMENTS == ZATTACHMENT.Z_PK
-            LEFT JOIN Z_2INTERACTIONRECIPIENT ON ZINTERACTIONS.Z_PK== Z_2INTERACTIONRECIPIENT.Z_3INTERACTIONRECIPIENT
-            LEFT JOIN ZCONTACTS RECEIPIENTCONACT ON Z_2INTERACTIONRECIPIENT.Z_2RECIPIENTS== RECEIPIENTCONACT.Z_PK;
-        """)
-        names = [description[0] for description in cur.description]
-
-        for item in cur:
-            self.results.append({
-                "start_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[0])),
-                "end_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[1])),
-                "bundle_id": item[2],
-                "account": item[3],
-                "target_bundle_id": item[4],
-                "direction": item[5],
-                "sender_display_name": item[6],
-                "sender_identifier": item[7],
-                "sender_personid": item[8],
-                "recipient_display_name": item[9],
-                "recipient_identifier": item[10],
-                "recipient_personid": item[11],
-                "recipient_count": item[12],
-                "domain_identifier": item[13],
-                "is_response": item[14],
-                "content": item[15],
-                "uti": item[16],
-                "content_url": item[17],
-                "size": item[18],
-                "photo_local_id": item[19],
-                "attachment_id": item[20],
-                "cloud_id": item[21],
-                "incoming_recipient_count": item[22],
-                "incoming_sender_count": item[23],
-                "outgoing_recipient_count": item[24],
-                "interactions_creation_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[25])) if item[25] else None,
-                "contacts_creation_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[26])) if item[26] else None,
-                "first_incoming_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[27])) if item[27] else None,
-                "first_incoming_sender_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[28])) if item[28] else None,
-                "first_outgoing_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[29])) if item[29] else None,
-                "last_incoming_sender_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[30])) if item[30] else None,
-                "last_incoming_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[31])) if item[31] else None,
-                "last_outgoing_recipient_date": convert_timestamp_to_iso(convert_mactime_to_unix(item[32])) if item[32] else None,
-                "custom_id": item[33],
-                "location_uuid": item[35],
-                "group_name": item[36],
-                "derivied_intent_id": item[37],
-                "table_id": item[38]
-            })
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d InteractionC events", len(self.results))

mvt/ios/modules/fs/locationd.py

@@ -1,69 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import os
-import glob
-import biplist
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-LOCATIOND_BACKUP_IDS = [
-    "a690d7769cce8904ca2b67320b107c8fe5f79412",
-]
-LOCATIOND_ROOT_PATHS = [
-    "private/var/mobile/Library/Caches/locationd/clients.plist",
-]
-
-class LocationdClients(IOSExtraction):
-    """Extract information from apps who used geolocation"""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-        self.timestamps = [
-            "ConsumptionPeriodBegin",
-            "ReceivingLocationInformationTimeStopped",
-            "VisitTimeStopped",
-            "LocationTimeStopped",
-            "BackgroundLocationTimeStopped",
-            "SignificantTimeStopped",
-            "NonPersistentSignificantTimeStopped",
-            "FenceTimeStopped",
-            "BeaconRegionTimeStopped",
-        ]
-
-    def serialize(self, record):
-        records = []
-        for ts in self.timestamps:
-            if ts in record.keys():
-                records.append({
-                    "timestamp": entry[ts],
-                    "module": self.__class__.__name__,
-                    "event": ts,
-                    "data": f"{ts} from {entry['package']}"
-                })
-
-        return records
-
-    def run(self):
-        self._find_ios_database(backup_ids=LOCATIOND_BACKUP_IDS, root_paths=LOCATIOND_ROOT_PATHS)
-        self.log.info("Found Locationd Clients plist at path: %s", self.file_path)
-        file_plist = biplist.readPlist(self.file_path)
-
-        for app in file_plist:
-            if file_plist[app] is dict:
-                result = file_plist[app]
-                result["package"] = app
-                for ts in self.timestamps:
-                    if ts in result.keys():
-                        result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[date]))
-
-                self.results.append(result)
-
-        self.log.info("Extracted a total of %d Locationd Clients entries", len(self.results))

mvt/ios/modules/fs/manifest.py

@@ -1,131 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import io
-import os
-import biplist
-import sqlite3
-import datetime
-
-from mvt.common.utils import convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-class Manifest(IOSExtraction):
-    """This module extracts information from a backup Manifest.db file."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def _get_key(self, dictionary, key):
-        """
-        Unserialized plist objects can have keys which are str or byte types
-
-        This is a helper to try fetch a key as both a byte or string type.
-        """
-        return dictionary.get(key.encode("utf-8"), None) or dictionary.get(key, None)
-
-    def _convert_timestamp(self, timestamp_or_unix_time_int):
-        """Older iOS versions stored the manifest times as unix timestamps."""
-        if isinstance(timestamp_or_unix_time_int, datetime.datetime):
-            return convert_timestamp_to_iso(timestamp_or_unix_time_int)
-        else:
-            timestamp = datetime.datetime.utcfromtimestamp(timestamp_or_unix_time_int)
-            return convert_timestamp_to_iso(timestamp)
-
-    def serialize(self, record):
-        records = []
-        for ts in set([record["created"], record["modified"], record["statusChanged"]]):
-            macb = ""
-            macb += "M" if ts == record["modified"] else "-"
-            macb += "-"
-            macb += "C" if ts == record["statusChanged"] else "-"
-            macb += "B" if ts == record["created"] else "-"
-
-            records.append({
-                "timestamp": ts,
-                "module": self.__class__.__name__,
-                "event": macb,
-                "data": f"{record['relativePath']} - {record['domain']}"
-            })
-
-        return records
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if not "relativePath" in result:
-                continue
-
-            if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain":
-                self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain")
-                self.detected.append(result)
-                continue
-
-            if self.indicators.check_file(result["relativePath"]):
-                self.log.warning("Found a known malicious file at path: %s", result["relativePath"])
-                self.detected.append(result)
-                continue
-
-            relPath = result["relativePath"].lower()
-            for ioc in self.indicators.ioc_domains:
-                if ioc.lower() in relPath:
-                    self.log.warning("Found mention of domain \"%s\" in a backup file with path: %s",
-                                     ioc, relPath)
-                    self.detected.append(result)
-
-    def run(self):
-        manifest_db_path = os.path.join(self.base_folder, "Manifest.db")
-        if not os.path.isfile(manifest_db_path):
-            raise FileNotFoundError("Impossible to find the module's database file")
-
-        self.log.info("Found Manifest.db database at path: %s", manifest_db_path)
-
-        conn = sqlite3.connect(manifest_db_path)
-        cur = conn.cursor()
-
-        cur.execute("SELECT * FROM Files;")
-        names = [description[0] for description in cur.description]
-
-        for file_entry in cur:
-            file_data = dict()
-            for index, value in enumerate(file_entry):
-                file_data[names[index]] = value
-
-            cleaned_metadata = {
-                "fileID": file_data["fileID"],
-                "domain": file_data["domain"],
-                "relativePath": file_data["relativePath"],
-                "flags": file_data["flags"],
-                "created": "",
-            }
-
-            if file_data["file"]:
-                try:
-                    file_plist = biplist.readPlist(io.BytesIO(file_data["file"]))
-                    file_metadata = self._get_key(file_plist, "$objects")[1]
-                    cleaned_metadata.update({
-                        "created": self._convert_timestamp(self._get_key(file_metadata, "Birth")),
-                        "modified": self._convert_timestamp(self._get_key(file_metadata, "LastModified")),
-                        "statusChanged": self._convert_timestamp(self._get_key(file_metadata, "LastStatusChange")),
-                        "mode": oct(self._get_key(file_metadata, "Mode")),
-                        "owner": self._get_key(file_metadata, "UserID"),
-                        "size": self._get_key(file_metadata, "Size"),
-                    })
-                except:
-                    self.log.exception("Error reading manifest file metadata")
-                    pass
-
-            self.results.append(cleaned_metadata)
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d file metadata items", len(self.results))

mvt/ios/modules/fs/net_base.py

@@ -1,224 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-import operator
-from pathlib import Path
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-class NetBase(IOSExtraction):
-    """This class provides a base for DataUsage and NetUsage extraction modules."""
-
-    def _extract_net_data(self):
-        conn = sqlite3.connect(self.file_path)
-        cur = conn.cursor()
-        cur.execute("""SELECT
-                ZPROCESS.ZFIRSTTIMESTAMP,
-                ZPROCESS.ZTIMESTAMP,
-                ZPROCESS.ZPROCNAME,
-                ZPROCESS.ZBUNDLENAME,
-                ZPROCESS.Z_PK,
-                ZLIVEUSAGE.ZWIFIIN,
-                ZLIVEUSAGE.ZWIFIOUT,
-                ZLIVEUSAGE.ZWWANIN,
-                ZLIVEUSAGE.ZWWANOUT,
-                ZLIVEUSAGE.Z_PK,
-                ZLIVEUSAGE.ZHASPROCESS,
-                ZLIVEUSAGE.ZTIMESTAMP
-            FROM ZLIVEUSAGE
-            LEFT JOIN ZPROCESS ON ZLIVEUSAGE.ZHASPROCESS = ZPROCESS.Z_PK;""")
-
-        items = []
-        for item in cur:
-            # ZPROCESS records can be missing after the JOIN. Handle NULL timestamps.
-            if item[0] and item[1]:
-                first_isodate = convert_timestamp_to_iso(convert_mactime_to_unix(item[0]))
-                isodate = convert_timestamp_to_iso(convert_mactime_to_unix(item[1]))
-            else:
-                first_isodate = item[0]
-                isodate = item[1]
-
-            if item[11]:
-                live_timestamp = convert_timestamp_to_iso(convert_mactime_to_unix(item[11]))
-            else:
-                live_timestamp = ""
-
-            items.append(dict(
-                first_isodate=first_isodate,
-                isodate=isodate,
-                proc_name=item[2],
-                bundle_id=item[3],
-                proc_id=item[4],
-                wifi_in=item[5],
-                wifi_out=item[6],
-                wwan_in=item[7],
-                wwan_out=item[8],
-                live_id=item[9],
-                live_proc_id=item[10],
-                live_isodate=live_timestamp,
-            ))
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted information on %d processes", len(items))
-        self.results = items
-
-    def serialize(self, record):
-        record_data = f"{record['proc_name']} (Bundle ID: {record['bundle_id']}, ID: {record['proc_id']})"
-        record_data_usage = record_data + f" WIFI IN: {record['wifi_in']}, WIFI OUT: {record['wifi_out']} - "  \
-                            f"WWAN IN: {record['wwan_in']}, WWAN OUT: {record['wwan_out']}"
-
-        records = [{
-            "timestamp": record["live_isodate"],
-            "module": self.__class__.__name__,
-            "event": "live_usage",
-            "data": record_data_usage,
-        }]
-
-        # Only included first_usage and current_usage records when a ZPROCESS entry exists.
-        if "MANIPULATED" not in record["proc_name"] and "MISSING" not in record["proc_name"]:
-            records.extend([
-                {
-                    "timestamp": record["first_isodate"],
-                    "module": self.__class__.__name__,
-                    "event": "first_usage",
-                    "data": record_data,
-                },
-                {
-                    "timestamp": record["isodate"],
-                    "module": self.__class__.__name__,
-                    "event": "current_usage",
-                    "data": record_data,
-                }
-            ])
-        return records
-
-    def _find_suspicious_processes(self):
-        if not self.is_fs_dump:
-            return
-
-        if not self.results:
-            return
-
-        # If we are instructed to run fast, we skip this.
-        if self.fast_mode:
-            self.log.info("Flag --fast was enabled: skipping extended search for suspicious processes")
-            return
-
-        self.log.info("Extended search for suspicious processes ...")
-
-        files = []
-        for posix_path in Path(self.base_folder).rglob("*"):
-            try:
-                if not posix_path.is_file():
-                    continue
-            except PermissionError:
-                continue
-
-            files.append([posix_path.name, posix_path.__str__()])
-
-        for proc in self.results:
-            if not proc["bundle_id"]:
-                self.log.debug("Found process with no Bundle ID with name: %s", proc["proc_name"])
-
-                binary_path = None
-                for file in files:
-                    if proc["proc_name"] == file[0]:
-                        binary_path = file[1]
-                        break
-
-                if binary_path:
-                    self.log.debug("Located at %s", binary_path)
-                else:
-                    msg = f"Could not find the binary associated with the process with name {proc['proc_name']}"
-                    if len(proc["proc_name"]) == 16:
-                        msg = msg + " (However, the process name might have been truncated in the database)"
-
-                    self.log.warning(msg)
-
-    def check_manipulated(self):
-        """Check for missing or manipulate DB entries
-        """
-        # Don't show duplicates for each missing process.
-        missing_process_cache = set()
-        for result in sorted(self.results, key=operator.itemgetter("live_isodate")):
-            if result["proc_id"]:
-                continue
-
-            # Avoid duplicate warnings for same process.
-            if result["live_proc_id"] not in missing_process_cache:
-                missing_process_cache.add(result["live_proc_id"])
-                self.log.warning("Found manipulated process entry %s. Entry on %s",
-                                 result["live_proc_id"], result["live_isodate"])
-
-            # Set manipulated proc timestamp so it appears in timeline.
-            result["first_isodate"] = result["isodate"] = result["live_isodate"]
-            result["proc_name"] = "MANIPULATED [process record deleted]"
-            self.detected.append(result)
-
-    def find_deleted(self):
-        """Identify process which may have been deleted from the DataUsage database"""
-        results_by_proc = {proc["proc_id"]: proc for proc in self.results if proc["proc_id"]}
-        all_proc_id = sorted(results_by_proc.keys())
-
-        missing_procs, last_proc_id = {}, None
-        for proc_id in range(min(all_proc_id), max(all_proc_id)):
-            if proc_id not in all_proc_id:
-                previous_proc = results_by_proc[last_proc_id]
-                self.log.info("Missing process %d. Previous process at \"%s\" (%s)",
-                              proc_id, previous_proc["first_isodate"], previous_proc["proc_name"])
-
-                missing_procs[proc_id] = {
-                    "proc_id": proc_id,
-                    "prev_proc_id": last_proc_id,
-                    "prev_proc_name": previous_proc["proc_name"],
-                    "prev_proc_bundle": previous_proc["bundle_id"],
-                    "prev_proc_first": previous_proc["first_isodate"],
-                }
-            else:
-                last_proc_id = proc_id
-
-        # Add a placeholder entry for the missing processes.
-        for proc_id, proc in missing_procs.items():
-            # Set default DataUsage keys.
-            result = {key: None for key in self.results[0].keys()}
-            result["first_isodate"] = result["isodate"] = result["live_isodate"] = proc["prev_proc_first"]
-            result["proc_name"] = "MISSING [follows {}]".format(proc["prev_proc_name"])
-            result["proc_id"] = result["live_proc_id"] = proc["proc_id"]
-            result["bundle_id"] = None
-
-            self.results.append(result)
-
-        self.results = sorted(self.results, key=operator.itemgetter("first_isodate"))
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            try:
-                proc_name = result["proc_name"]
-            except KeyError:
-                continue
-
-            # Process ID may be empty if process records have been manipulated.
-            if not result["proc_id"]:
-                continue
-
-            if self.indicators.check_process(proc_name):
-                self.detected.append(result)
-
-        # Check for manipulated process records.
-        # TODO: Catching KeyError for live_isodate for retro-compatibility.
-        #       This is not very good.
-        try:
-            self.check_manipulated()
-            self.find_deleted()
-        except KeyError:
-            pass

mvt/ios/modules/fs/net_datausage.py

@@ -1,30 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .net_base import NetBase
-
-DATAUSAGE_BACKUP_IDS = [
-    "0d609c54856a9bb2d56729df1d68f2958a88426b",
-]
-DATAUSAGE_ROOT_PATHS = [
-    "private/var/wireless/Library/Databases/DataUsage.sqlite",
-]
-
-class Datausage(NetBase):
-    """This class extracts data from DataUsage.sqlite and attempts to identify
-    any suspicious processes if running on a full filesystem dump."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._find_ios_database(backup_ids=DATAUSAGE_BACKUP_IDS, root_paths=DATAUSAGE_ROOT_PATHS)
-        self.log.info("Found DataUsage database at path: %s", self.file_path)
-
-        self._extract_net_data()
-        self._find_suspicious_processes()

mvt/ios/modules/fs/net_netusage.py

@@ -1,28 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-from .net_base import NetBase
-
-NETUSAGE_ROOT_PATHS = [
-    "private/var/networkd/netusage.sqlite",
-    "private/var/networkd/db/netusage.sqlite"
-]
-
-class Netusage(NetBase):
-    """This class extracts data from netusage.sqlite and attempts to identify
-    any suspicious processes if running on a full filesystem dump."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def run(self):
-        self._find_ios_database(root_paths=NETUSAGE_ROOT_PATHS)
-        self.log.info("Found NetUsage database at path: %s", self.file_path)
-
-        self._extract_net_data()
-        self._find_suspicious_processes()

mvt/ios/modules/fs/safari_browserstate.py

@@ -1,103 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import io
-import biplist
-import sqlite3
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-from mvt.common.utils import keys_bytes_to_string
-
-from .base import IOSExtraction
-
-SAFARI_BROWSER_STATE_BACKUP_IDS = [
-    "3a47b0981ed7c10f3e2800aa66bac96a3b5db28e",
-]
-SAFARI_BROWSER_STATE_ROOT_PATHS = [
-    "private/var/mobile/Library/Safari/BrowserState.db",
-    "private/var/mobile/Containers/Data/Application/*/Library/Safari/BrowserState.db",
-]
-
-class SafariBrowserState(IOSExtraction):
-    """This module extracts all Safari browser state records."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["last_viewed_timestamp"],
-            "module": self.__class__.__name__,
-            "event": "tab",
-            "data": f"{record['tab_title']} - {record['tab_url']}"
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if "tab_url" in result and self.indicators.check_domain(result["tab_url"]):
-                self.detected.append(result)
-                continue
-
-            if not "session_data" in result:
-                continue
-
-            for session_entry in result["session_data"]:
-                if "entry_url" in session_entry and self.indicators.check_domain(session_entry["entry_url"]):
-                    self.detected.append(result)
-
-    def run(self):
-        self._find_ios_database(backup_ids=SAFARI_BROWSER_STATE_BACKUP_IDS,
-                                root_paths=SAFARI_BROWSER_STATE_ROOT_PATHS)
-        self.log.info("Found Safari browser state database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-
-        # Fetch valid icon cache.
-        cur = conn.cursor()
-        cur.execute("""SELECT
-                tabs.title,
-                tabs.url,
-                tabs.user_visible_url,
-                tabs.last_viewed_time,
-                tab_sessions.session_data
-            FROM tabs
-            JOIN tab_sessions ON tabs.uuid = tab_sessions.tab_uuid
-            ORDER BY tabs.last_viewed_time;""")
-
-        session_history_count = 0
-        for item in cur:
-            session_entries = []
-
-            if item[4]:
-                # Skip a 4 byte header before the plist content.
-                session_plist = item[4][4:]
-                session_data = biplist.readPlist(io.BytesIO(session_plist))
-                session_data = keys_bytes_to_string(session_data)
-
-                if "SessionHistoryEntries" in session_data["SessionHistory"]:
-                    for session_entry in session_data["SessionHistory"]["SessionHistoryEntries"]:
-                        session_history_count += 1
-                        session_entries.append(dict(
-                            entry_title=session_entry["SessionHistoryEntryOriginalURL"],
-                            entry_url=session_entry["SessionHistoryEntryURL"],
-                            data_length=len(session_entry["SessionHistoryEntryData"]) if "SessionHistoryEntryData" in session_entry else 0,
-                        ))
-
-            self.results.append(dict(
-                tab_title=item[0],
-                tab_url=item[1],
-                tab_visible_url=item[2],
-                last_viewed_timestamp=convert_timestamp_to_iso(convert_mactime_to_unix(item[3])),
-                session_data=session_entries,
-            ))
-
-        self.log.info("Extracted a total of %d tab records and %d session history entries",
-                      len(self.results), session_history_count)

mvt/ios/modules/fs/safari_favicon.py

@@ -1,88 +0,0 @@
-# Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021 MVT Project Developers.
-# See the file 'LICENSE' for usage and copying permissions, or find a copy at
-#   https://github.com/mvt-project/mvt/blob/main/LICENSE
-
-import sqlite3
-
-from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
-
-from .base import IOSExtraction
-
-SAFARI_FAVICON_ROOT_PATHS = [
-    "private/var/mobile/Library/Image Cache/Favicons/Favicons.db",
-    "private/var/mobile/Containers/Data/Application/*/Library/Image Cache/Favicons/Favicons.db",
-]
-
-class SafariFavicon(IOSExtraction):
-    """This module extracts all Safari favicon records."""
-
-    def __init__(self, file_path=None, base_folder=None, output_folder=None,
-                 fast_mode=False, log=None, results=[]):
-        super().__init__(file_path=file_path, base_folder=base_folder,
-                         output_folder=output_folder, fast_mode=fast_mode,
-                         log=log, results=results)
-
-    def serialize(self, record):
-        return {
-            "timestamp": record["isodate"],
-            "module": self.__class__.__name__,
-            "event": "safari_favicon",
-            "data": f"Safari favicon from {record['url']} with icon URL {record['icon_url']} ({record['type']})",
-        }
-
-    def check_indicators(self):
-        if not self.indicators:
-            return
-
-        for result in self.results:
-            if self.indicators.check_domain(result["url"]) or self.indicators.check_domain(result["icon_url"]):
-                self.detected.append(result)
-
-    def run(self):
-        self._find_ios_database(root_paths=SAFARI_FAVICON_ROOT_PATHS)
-        self.log.info("Found Safari favicon cache database at path: %s", self.file_path)
-
-        conn = sqlite3.connect(self.file_path)
-
-        # Fetch valid icon cache.
-        cur = conn.cursor()
-        cur.execute("""SELECT
-                page_url.url,
-                icon_info.url,
-                icon_info.timestamp
-            FROM page_url
-            JOIN icon_info ON page_url.uuid = icon_info.uuid
-            ORDER BY icon_info.timestamp;""")
-
-        items = []
-        for item in cur:
-            items.append(dict(
-                url=item[0],
-                icon_url=item[1],
-                timestamp=item[2],
-                isodate=convert_timestamp_to_iso(convert_mactime_to_unix(item[2])),
-                type="valid",
-            ))
-
-        # Fetch icons from the rejected icons table.
-        cur.execute("""SELECT
-                page_url,
-                icon_url,
-                timestamp
-            FROM rejected_resources ORDER BY timestamp;""")
-
-        for item in cur:
-            items.append(dict(
-                url=item[0],
-                icon_url=item[1],
-                timestamp=item[2],
-                isodate=convert_timestamp_to_iso(convert_mactime_to_unix(item[2])),
-                type="rejected",
-            ))
-
-        cur.close()
-        conn.close()
-
-        self.log.info("Extracted a total of %d favicon records", len(items))
-        self.results = sorted(items, key=lambda item: item["isodate"])