Bumped version

Create adb keys (Fixes #165)

Dockerfile

@@ -2,48 +2,56 @@ FROM ubuntu:20.04
 
 # Ref. https://github.com/mvt-project/mvt
 
+LABEL url="https://mvt.re"
+LABEL vcs-url="https://github.com/mvt-project/mvt"
+LABEL description="MVT is a forensic tool to look for signs of infection in smartphone devices."
+
+ENV PIP_NO_CACHE_DIR=1
+
 # Fixing major OS dependencies
 # ----------------------------
 RUN apt update \
   && apt install -y python3 python3-pip libusb-1.0-0-dev \
-  && apt install -y wget \ 
-  && apt install -y adb \ 
-  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless
+  && apt install -y wget unzip\ 
+  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless \
 
 # Install build tools for libimobiledevice
 # ----------------------------------------
-RUN apt install -y build-essential \
-	checkinstall \
-	git \
-	autoconf \
-	automake \
-	libtool-bin \
-	libplist-dev \
-	libusbmuxd-dev \
-	libssl-dev \
-	sqlite3 \
-    pkg-config
+  build-essential \
+  checkinstall \
+  git \
+  autoconf \
+  automake \
+  libtool-bin \
+  libplist-dev \
+  libusbmuxd-dev \
+  libssl-dev \
+  sqlite3 \
+  pkg-config \
 
 # Clean up
 # --------
-RUN apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /var/cache/apt
 
 
 # Build libimobiledevice
 # ----------------------
-RUN git clone https://github.com/libimobiledevice/libplist
-RUN git clone https://github.com/libimobiledevice/libusbmuxd
-RUN git clone https://github.com/libimobiledevice/libimobiledevice
-RUN git clone https://github.com/libimobiledevice/usbmuxd
+RUN git clone https://github.com/libimobiledevice/libplist \
+  && git clone https://github.com/libimobiledevice/libusbmuxd \
+  && git clone https://github.com/libimobiledevice/libimobiledevice \
+  && git clone https://github.com/libimobiledevice/usbmuxd \
 
-RUN cd libplist && ./autogen.sh && make && make install && ldconfig
+  && cd libplist && ./autogen.sh && make && make install && ldconfig \
 
-RUN cd libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig
+  && cd ../libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig \
 
-RUN cd libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig
+  && cd ../libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig \
 
-RUN cd usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install
+  && cd ../usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install \
+
+  # Clean up.
+  && cd .. && rm -rf libplist libusbmuxd libimobiledevice usbmuxd
 
 # Installing MVT
 # --------------
@@ -51,16 +59,29 @@ RUN pip3 install mvt
 
 # Installing ABE
 # --------------
-RUN mkdir /opt/abe
-RUN wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar
+RUN mkdir /opt/abe \
+  && wget https://github.com/nelenkov/android-backup-extractor/releases/download/20210709062403-4c55371/abe.jar -O /opt/abe/abe.jar \
 # Create alias for abe
-RUN echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+  && echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
+
+# Install Android Platform Tools 
+# ------------------------------ 
+
+RUN mkdir /opt/android \
+  && wget -q https://dl.google.com/android/repository/platform-tools-latest-linux.zip \
+  && unzip platform-tools-latest-linux.zip -d /opt/android \
+# Create alias for adb 
+  && echo 'alias adb="/opt/android/platform-tools/adb"' >> ~/.bashrc 
+
+# Generate adb key folder 
+# ------------------------------ 
+RUN mkdir /root/.android && /opt/android/platform-tools/adb keygen /root/.android/adbkey
 
 # Setup investigations environment
 # --------------------------------
 RUN mkdir /home/cases
 WORKDIR /home/cases 
-RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc
-RUN echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
+RUN echo 'echo "Mobile Verification Toolkit @ Docker\n------------------------------------\n\nYou can find information about how to use this image for Android (https://github.com/mvt-project/mvt/tree/master/docs/android) and iOS (https://github.com/mvt-project/mvt/tree/master/docs/ios) in the official docs of the project.\n"' >> ~/.bashrc \
+  && echo 'echo "Note that to perform the debug via USB you might need to give the Docker image access to the USB using \"docker run -it --privileged -v /dev/bus/usb:/dev/bus/usb mvt\" or, preferably, the \"--device=\" parameter.\n"' >> ~/.bashrc
 
 CMD /bin/bash

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-     <img src="./docs/mvt.png" width="300" />
+     <img src="./docs/mvt.png" width="200" />
 </p>
 
 # Mobile Verification Toolkit
@@ -27,7 +27,7 @@ Alternatively, you can decide to run MVT and all relevant tools through a [Docke
 
 ## Usage
 
-MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/).
+MVT provides two commands `mvt-ios` and `mvt-android`. [Check out the documentation to learn how to use them!](https://docs.mvt.re/)
 
 ## License
 

docs/android/adb.md

@@ -1,8 +1,42 @@
 # Check over ADB
 
-TODO
+In order to check an Android device over the [Android Debug Bridge (adb)](https://developer.android.com/studio/command-line/adb) you will first need to install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools). If you have installed [Android Studio](https://developer.android.com/studio/) you should already have access to `adb` and other utilities.
 
-<!-- In order to use `mvt-android` you need to connect your Android device to your computer. You will then need to [enable USB debugging](https://developer.android.com/studio/debug/dev-options#enable>) on the Android device.
+While many Linux distributions already package Android Platform Tools (for example `android-platform-tools-base` on Debian), it is preferable to install the most recent version from the official website. Packaged versions might be outdated and incompatible with most recent Android handsets.
 
-If this is the first time you connect to this device, you will need to approve the authentication keys through a prompt that will appear on your Android device.
- -->
+Next you will need to enable debugging on the Android device you are testing. [Please follow the official instructions on how to do so.](https://developer.android.com/studio/command-line/adb)
+
+## Connecting over USB
+
+The easiest way to check the device is over a USB transport. You will need to have USB debugging enabled and the device plugged into your computer. If everything is configured appropriately you should see your device when launching the command `adb devices`.
+
+Now you can try launching MVT with:
+
+```bash
+mvt-android check-adb --output /path/to/results
+```
+
+If you have previously started an adb daemon MVT will alert you and require you to kill it with `adb kill-server` and relaunch the command.
+
+!!! warning
+    MVT relies on the Python library [adb-shell](https://pypi.org/project/adb-shell/) to connect to an Android device, which relies on libusb for the USB transport. Because of known driver issues, Windows users [are recommended](https://github.com/JeffLIrion/adb_shell/issues/118) to install appropriate drivers using [Zadig](https://zadig.akeo.ie/). Alternatively, an easier option might be to use the TCP transport and connect over Wi-Fi as describe next.
+
+## Connecting over Wi-FI
+
+When connecting to the device over USB is not possible or not working properly, an alternative option is to connect over the network. In order to do so, first launch an adb daemon at a fixed port number:
+
+```bash
+adb tcpip 5555
+```
+
+Then you can specify the IP address of the phone with the adb port number to MVT like so:
+
+```bash
+mvt-android check-adb --serial 192.168.1.20:5555 --output /path/to/results
+```
+
+Where `192.168.1.20` is the correct IP address of your device.
+
+## MVT modules requiring root privileges
+
+Of the currently available `mvt-android check-adb` modules a handful require root privileges to function correctly. This is because certain files, such as browser history and SMS messages databases are not accessible with user privileges through adb. These modules are to be considered OPTIONALLY available in case the device was already jailbroken. **Do NOT jailbreak your own device unless you are sure of what you are doing!** Jailbreaking your phone exposes it to considerable security risks!

docs/android/backup.md

@@ -44,4 +44,4 @@ $ mvt-android check-backup --output /path/to/results/ /path/to/backup/
                   64 SMS messages containing links
 ```
 
-Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by mvt. Any matches will be highlighted in the terminal output.
+Through the `--iocs` argument you can specify a [STIX2](https://oasis-open.github.io/cti-documentation/stix/intro) file defining a list of malicious indicators to check against the records extracted from the backup by MVT. Any matches will be highlighted in the terminal output.

docs/android/download_apks.md

@@ -20,7 +20,7 @@ mvt-android download-apks --output /path/to/folder --virustotal
 mvt-android download-apks --output /path/to/folder --koodous
 ```
 
-Or, to launch all available lookups::
+Or, to launch all available lookups:
 
 ```bash
 mvt-android download-apks --output /path/to/folder --all-checks

docs/android/methodology.md

@@ -13,8 +13,9 @@ While it is out of the scope of this documentation to dwell into details on how
 
 ## Check the device over Android Debug Bridge
 
-TODO
+Some additional diagnostic information can be extracted from the phone using the [Android Debug Bridge (adb)](https://developer.android.com/studio/command-line/adb). `mvt-android` allows to automatically extract information including [dumpsys](https://developer.android.com/studio/command-line/dumpsys) results, details on installed packages (without download), running processes, presence of root binaries and packages, and more.
+
 
 ## Check an Android Backup (SMS messages)
 
-TODO
+Although Android backups are becoming deprecated, it is still possible to generate one. Unfortunately, because apps these days typically favor backup over the cloud, the amount of data available is limited. Currently, `mvt-android check-backup` only supports checking SMS messages containing links.

docs/index.md

@@ -10,4 +10,4 @@ In this documentation you will find instructions on how to install and run the `
 
 ## Resources
 
-[:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button }
+[:fontawesome-brands-github: GitHub](https://github.com/mvt-project/mvt){: .md-button .md-button--primary } [:fontawesome-brands-python: Python Package](https://pypi.org/project/mvt){: .md-button }

docs/install.md

@@ -1,6 +1,6 @@
 # Installation
 
-Before proceeding, please note that mvt requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
+Before proceeding, please note that MVT requires Python 3.6+ to run. While it should be available on most operating systems, please make sure of that before proceeding.
 
 ## Dependencies on Linux
 
@@ -14,9 +14,9 @@ sudo apt install python3 python3-pip libusb-1.0-0 sqlite3
 
 When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools). If you prefer to install a package made available by your distribution of choice, please make sure the version is recent to ensure compatibility with modern Android devices.
 
-## Dependencies on Mac
+## Dependencies on macOS
 
-Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installed.
+Running MVT on macOS requires Xcode and [homebrew](https://brew.sh) to be installed.
 
 In order to install dependencies use:
 
@@ -26,7 +26,7 @@ brew install python3 libusb sqlite3
 
 *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
 
-When working with Android devices you should additionally install Android SDK Platform Tools:
+When working with Android devices you should additionally install [Android SDK Platform Tools](https://developer.android.com/studio/releases/platform-tools):
 
 ```bash
 brew install --cask android-platform-tools

docs/introduction.md

@@ -10,6 +10,8 @@ Mobile Verification Toolkit (MVT) is a collection of utilities designed to facil
 - Generate JSON logs of extracted records, and separate JSON logs of all detected malicious traces.
 - Generate a unified chronological timeline of extracted records, along with a timeline all detected malicious traces.
 
+MVT is a forensic research tool intended for technologists and investigators. Using it requires understanding the basics of forensic analysis and using command-line tools. MVT is not intended for end-user self-assessment. If you are concerned with the security of your device please seek expert assistance.
+
 ## Consensual Forensics
 
 While MVT is capable of extracting and processing various types of very personal records typically found on a mobile phone (such as calls history, SMS and WhatsApp messages, etc.), this is intended to help identify potential attack vectors such as malicious SMS messages leading to exploitation.

docs/ios/backup/itunes.md

@@ -1,16 +1,16 @@
 # Backup with iTunes app
 
-It is possible to do an iPhone backup by using iTunes on Windows or Mac computers (in most recent versions of Mac OS, this feature is included in Finder).
+It is possible to do an iPhone backup by using iTunes on Windows or macOS computers (in most recent versions of macOS, this feature is included in Finder).
 
 To do that:
 
 * Make sure iTunes is installed.
 * Connect your iPhone to your computer using a Lightning/USB cable.
-* Open the device in iTunes (or Finder on Mac OS).
+* Open the device in iTunes (or Finder on macOS).
 * If you want to have a more accurate detection, ensure that the encrypted backup option is activated and choose a secure password for the backup.
 * Start the backup and wait for it to finish (this may take up to 30 minutes).
 
 ![](../../../img/macos-backup.jpg)
 _Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
 
-* Once the backup is done, find its location and copy it to a place where it can be analyzed by `mvt`. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On Mac OS, the backup is stored in `~/Library/Application Support/MobileSync/`.
+* Once the backup is done, find its location and copy it to a place where it can be analyzed by MVT. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On macOS, the backup is stored in `~/Library/Application Support/MobileSync/`.

docs/ios/methodology.md

@@ -12,4 +12,4 @@ If you are not expected to return the phone, you might want to consider to attem
 
 #### iTunes Backup
 
-An alternative option is to generate an iTunes backup (in most recent version of mac OS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc.
+An alternative option is to generate an iTunes backup (in most recent version of macOS, they are no longer launched from iTunes, but directly from Finder). While backups only provide a subset of the files stored on the device, in many cases it might be sufficient to at least detect some suspicious artifacts. Backups encrypted with a password will have some additional interesting records not available in unencrypted ones, such as Safari history, Safari state, etc.

docs/ios/records.md

@@ -29,7 +29,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `calls.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Calls` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CallHistoryDB/CallHistory.storedata*, which contains records of incoming and outgoing calls, including from messaging apps such as WhatsApp or Skype.
@@ -107,17 +107,19 @@ If indicators a provided through the command-line, they are checked against the
 ### `id_status_cache.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (before iOS 14.7): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `IDStatusCache` module. The module extracts records from a plist file located at */private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist*, which contains a cache of Apple user ID authentication. This chance will indicate when apps like Facetime and iMessage first established contacts with other registered Apple IDs. This is significant because it might contain traces of malicious accounts involved in exploitation of those apps.
 
+Starting from iOS 14.7.0, this file is empty or absent.
+
 ---
 
 ### `interaction_c.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `InteractionC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CoreDuet/People/interactionC.db*, which contains details about user interactions with installed apps.
@@ -183,7 +185,7 @@ This JSON file is created by mvt-ios' `ProfileEvents` module. The module extract
 ### `safari_browser_state.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariBrowserState` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/BrowserState.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/BrowserState.db*, which contain records of opened tabs.
@@ -207,7 +209,7 @@ If indicators are provided through the command-line, they are checked against bo
 ### `safari_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup (if encrypted): :material-check:  
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariHistory` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/History.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/History.db*, which contain a history of URL visits.

mvt/android/cli.py

@@ -9,6 +9,7 @@ import os
 import click
 from rich.logging import RichHandler
 
+from mvt.common.help import *
 from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
 from mvt.common.module import run_module, save_timeline
 
@@ -24,10 +25,6 @@ logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
     RichHandler(show_path=False, log_time_format="%X")])
 log = logging.getLogger(__name__)
 
-# Help messages of repeating options.
-OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results"
-SERIAL_HELP_MESSAGE = "Specify a device serial number or HOST:PORT connection string"
-
 #==============================================================================
 # Main
 #==============================================================================
@@ -40,9 +37,9 @@ def cli():
 # Download APKs
 #==============================================================================
 @cli.command("download-apks", help="Download all or non-safelisted installed APKs installed on the device")
-@click.option("--serial", "-s", type=str, help=SERIAL_HELP_MESSAGE)
+@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
 @click.option("--all-apks", "-a", is_flag=True,
-              help="Extract all packages installed on the phone, even those marked as safe")
+              help="Extract all packages installed on the phone, including system packages")
 @click.option("--virustotal", "-v", is_flag=True, help="Check packages on VirusTotal")
 @click.option("--koodous", "-k", is_flag=True, help="Check packages on Koodous")
 @click.option("--all-checks", "-A", is_flag=True, help="Run all available checks")
@@ -68,7 +65,8 @@ def download_apks(ctx, all_apks, virustotal, koodous, all_checks, output, from_f
                     log.critical("Unable to create output folder %s: %s", output, e)
                     ctx.exit(1)
 
-            download = DownloadAPKs(output_folder=output, all_apks=all_apks)
+            download = DownloadAPKs(output_folder=output, all_apks=all_apks,
+                                    log=logging.getLogger(DownloadAPKs.__module__))
             if serial:
                 download.serial = serial
             download.run()
@@ -92,13 +90,13 @@ def download_apks(ctx, all_apks, virustotal, koodous, all_checks, output, from_f
 # Checks through ADB
 #==============================================================================
 @cli.command("check-adb", help="Check an Android device over adb")
-@click.option("--serial", "-s", type=str, help=SERIAL_HELP_MESSAGE)
+@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
-              default=[], help="Path to indicators file (can be invoked multiple times)")
+              default=[], help=HELP_MSG_IOC)
 @click.option("--output", "-o", type=click.Path(exists=False),
-              help="Specify a path to a folder where you want to store JSON results")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
+              help=HELP_MSG_OUTPUT)
+@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
+@click.option("--module", "-m", help=HELP_MSG_MODULE)
 @click.pass_context
 def check_adb(ctx, iocs, output, list_modules, module, serial):
     if list_modules:
@@ -155,10 +153,10 @@ def check_adb(ctx, iocs, output, list_modules, module, serial):
 # Check ADB backup
 #==============================================================================
 @cli.command("check-backup", help="Check an Android Backup")
-@click.option("--serial", "-s", type=str, help=SERIAL_HELP_MESSAGE)
+@click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
-              default=[], help="Path to indicators file (can be invoked multiple times)")
-@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE)
+              default=[], help=HELP_MSG_IOC)
+@click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_backup(ctx, iocs, output, backup_path, serial):

mvt/android/data/safe_packages.txt

@@ -1,182 +0,0 @@
-android
-android.auto_generated_rro__
-android.autoinstalls.config.google.nexus
-com.android.backupconfirm
-com.android.bips
-com.android.bluetooth
-com.android.bluetoothmidiservice
-com.android.bookmarkprovider
-com.android.calllogbackup
-com.android.captiveportallogin
-com.android.carrierconfig
-com.android.carrierdefaultapp
-com.android.cellbroadcastreceiver
-com.android.certinstaller
-com.android.chrome
-com.android.companiondevicemanager
-com.android.connectivity.metrics
-com.android.cts.ctsshim
-com.android.cts.priv.ctsshim
-com.android.defcontainer
-com.android.documentsui
-com.android.dreams.basic
-com.android.egg
-com.android.emergency
-com.android.externalstorage
-com.android.facelock
-com.android.hotwordenrollment
-com.android.hotwordenrollment.okgoogle
-com.android.hotwordenrollment.tgoogle
-com.android.hotwordenrollment.xgoogle
-com.android.htmlviewer
-com.android.inputdevices
-com.android.keychain
-com.android.location.fused
-com.android.managedprovisioning
-com.android.mms.service
-com.android.mtp
-com.android.musicfx
-com.android.nfc
-com.android.omadm.service
-com.android.pacprocessor
-com.android.phone
-com.android.printspooler
-com.android.providers.blockednumber
-com.android.providers.calendar
-com.android.providers.contacts
-com.android.providers.downloads
-com.android.providers.downloads.ui
-com.android.providers.media
-com.android.providers.partnerbookmarks
-com.android.providers.settings
-com.android.providers.telephony
-com.android.providers.userdictionary
-com.android.proxyhandler
-com.android.retaildemo
-com.android.safetyregulatoryinfo
-com.android.sdm.plugins.connmo
-com.android.sdm.plugins.dcmo
-com.android.sdm.plugins.diagmon
-com.android.sdm.plugins.sprintdm
-com.android.server.telecom
-com.android.service.ims
-com.android.service.ims.presence
-com.android.settings
-com.android.sharedstoragebackup
-com.android.shell
-com.android.statementservice
-com.android.stk
-com.android.systemui
-com.android.systemui.theme.dark
-com.android.vending
-com.android.vpndialogs
-com.android.vzwomatrigger
-com.android.wallpaperbackup
-com.android.wallpaper.livepicker
-com.breel.wallpapers
-com.customermobile.preload.vzw
-com.google.android.apps.cloudprint
-com.google.android.apps.docs
-com.google.android.apps.docs.editors.docs
-com.google.android.apps.enterprise.dmagent
-com.google.android.apps.gcs
-com.google.android.apps.helprtc
-com.google.android.apps.inputmethod.hindi
-com.google.android.apps.maps
-com.google.android.apps.messaging
-com.google.android.apps.nexuslauncher
-com.google.android.apps.photos
-com.google.android.apps.pixelmigrate
-com.google.android.apps.tachyon
-com.google.android.apps.turbo
-com.google.android.apps.tycho
-com.google.android.apps.wallpaper
-com.google.android.apps.wallpaper.nexus
-com.google.android.apps.work.oobconfig
-com.google.android.apps.youtube.vr
-com.google.android.asdiv
-com.google.android.backuptransport
-com.google.android.calculator
-com.google.android.calendar
-com.google.android.carrier
-com.google.android.carrier.authdialog
-com.google.android.carrierentitlement
-com.google.android.carriersetup
-com.google.android.configupdater
-com.google.android.contacts
-com.google.android.deskclock
-com.google.android.dialer
-com.google.android.euicc
-com.google.android.ext.services
-com.google.android.ext.shared
-com.google.android.feedback
-com.google.android.gm
-com.google.android.gms
-com.google.android.gms.policy_auth
-com.google.android.gms.policy_sidecar_o
-com.google.android.gms.setup
-com.google.android.GoogleCamera
-com.google.android.googlequicksearchbox
-com.google.android.gsf
-com.google.android.gsf.login
-com.google.android.hardwareinfo
-com.google.android.hiddenmenu
-com.google.android.ims
-com.google.android.inputmethod.japanese
-com.google.android.inputmethod.korean
-com.google.android.inputmethod.latin
-com.google.android.inputmethod.pinyin
-com.google.android.instantapps.supervisor
-com.google.android.keep
-com.google.android.marvin.talkback
-com.google.android.music
-com.google.android.nexusicons
-com.google.android.onetimeinitializer
-com.google.android.packageinstaller
-com.google.android.partnersetup
-com.google.android.printservice.recommendation
-com.google.android.setupwizard
-com.google.android.soundpicker
-com.google.android.storagemanager
-com.google.android.syncadapters.contacts
-com.google.android.tag
-com.google.android.talk
-com.google.android.tetheringentitlement
-com.google.android.theme.pixel
-com.google.android.tts
-com.google.android.videos
-com.google.android.vr.home
-com.google.android.vr.inputmethod
-com.google.android.webview
-com.google.android.wfcactivation
-com.google.android.youtube
-com.google.ar.core
-com.google.intelligence.sense
-com.google.modemservice
-com.google.pixel.wahoo.gfxdrv
-com.google.SSRestartDetector
-com.google.tango
-com.google.vr.apps.ornament
-com.google.vr.vrcore
-com.htc.omadm.trigger
-com.qti.qualcomm.datastatusnotification
-com.qualcomm.atfwd
-com.qualcomm.embms
-com.qualcomm.fastdormancy
-com.qualcomm.ltebc_vzw
-com.qualcomm.qcrilmsgtunnel
-com.qualcomm.qti.ims
-com.qualcomm.qti.networksetting
-com.qualcomm.qti.telephonyservice
-com.qualcomm.qti.uceShimService
-com.qualcomm.shutdownlistner
-com.qualcomm.timeservice
-com.qualcomm.vzw_api
-com.quicinc.cne.CNEService
-com.verizon.llkagent
-com.verizon.mips.services
-com.verizon.obdm
-com.verizon.obdm_permissions
-com.verizon.services
-com.vzw.apnlib
-qualcomm.com.vzw_msdc_api

mvt/android/download_apks.py

@@ -14,6 +14,7 @@ from mvt.common.module import InsufficientPrivileges
 from mvt.common.utils import get_sha256_from_file_path
 
 from .modules.adb.base import AndroidExtraction
+from .modules.adb.packages import Packages
 
 log = logging.getLogger(__name__)
 
@@ -29,33 +30,23 @@ class PullProgress(tqdm):
         self.update(current - self.n)
 
 
-class Package:
-    """Package indicates a package name and all the files associated with it."""
-
-    def __init__(self, name, files=None):
-        self.name = name
-        self.files = files or []
-
-
 class DownloadAPKs(AndroidExtraction):
     """DownloadAPKs is the main class operating the download of APKs
     from the device."""
 
-    def __init__(self, output_folder=None, all_apks=False, packages=None):
+    def __init__(self, output_folder=None, all_apks=False, log=None,
+                 packages=None):
         """Initialize module.
         :param output_folder: Path to the folder where data should be stored
         :param all_apks: Boolean indicating whether to download all packages
                          or filter known-goods
         :param packages: Provided list of packages, typically for JSON checks
         """
-        super().__init__(file_path=None, base_folder=None,
-                         output_folder=output_folder)
+        super().__init__(output_folder=output_folder, log=log)
 
-        self.output_folder_apk = None
-        self.packages = packages or []
+        self.packages = packages
         self.all_apks = all_apks
-
-        self._safe_packages = []
+        self.output_folder_apk = None
 
     @classmethod
     def from_json(cls, json_path):
@@ -63,55 +54,9 @@ class DownloadAPKs(AndroidExtraction):
         :param json_path: Path to the apks.json file to parse.
         """
         with open(json_path, "r") as handle:
-            data = json.load(handle)
-
-            packages = []
-            for entry in data:
-                package = Package(entry["name"], entry["files"])
-                packages.append(package)
-
+            packages = json.load(handle)
             return cls(packages=packages)
 
-    def _load_safe_packages(self):
-        """Load known-good package names.
-        """
-        safe_packages_path = os.path.join("data", "safe_packages.txt")
-        safe_packages_string = pkg_resources.resource_string(__name__, safe_packages_path)
-        safe_packages_list = safe_packages_string.decode("utf-8").split("\n")
-        self._safe_packages.extend(safe_packages_list)
-
-    def _clean_output(self, output):
-        """Clean adb shell command output.
-        :param output: Command output to clean.
-        """
-        return output.strip().replace("package:", "")
-
-    def get_packages(self):
-        """Retrieve package names from the device using adb.
-        """
-        log.info("Retrieving package names ...")
-
-        if not self.all_apks:
-            self._load_safe_packages()
-
-        output = self._adb_command("pm list packages")
-        total = 0
-        for line in output.split("\n"):
-            package_name = self._clean_output(line)
-            if package_name == "":
-                continue
-
-            total += 1
-
-            if not self.all_apks and package_name in self._safe_packages:
-                continue
-
-            if package_name not in self.packages:
-                self.packages.append(Package(package_name))
-
-        log.info("There are %d packages installed on the device. I selected %d for inspection.",
-                 total, len(self.packages))
-
     def pull_package_file(self, package_name, remote_path):
         """Pull files related to specific package from the device.
         :param package_name: Name of the package to download
@@ -153,6 +98,18 @@ class DownloadAPKs(AndroidExtraction):
 
         return local_path
 
+    def get_packages(self):
+        """Use the Packages adb module to retrieve the list of packages.
+        We reuse the same extraction logic to then download the APKs.
+        """
+        self.log.info("Retrieving list of installed packages...")
+
+        m = Packages()
+        m.log = self.log
+        m.run()
+
+        self.packages = m.results
+
     def pull_packages(self):
         """Download all files of all selected packages from the device.
         """
@@ -161,26 +118,47 @@ class DownloadAPKs(AndroidExtraction):
         if not os.path.exists(self.output_folder):
             os.mkdir(self.output_folder)
 
+        # If the user provided the flag --all-apks we select all packages.
+        packages_selection = []
+        if self.all_apks:
+            log.info("Selected all %d available packages", len(self.packages))
+            packages_selection = self.packages
+        else:
+            # Otherwise we loop through the packages and get only those that
+            # are not marked as system.
+            for package in self.packages:
+                if not package.get("system", False):
+                    packages_selection.append(package)
+
+            log.info("Selected only %d packages which are not marked as system",
+                len(packages_selection))
+
+        if len(packages_selection) == 0:
+            log.info("No packages were selected for download")
+            return
+
         log.info("Downloading packages from device. This might take some time ...")
 
         self.output_folder_apk = os.path.join(self.output_folder, "apks")
         if not os.path.exists(self.output_folder_apk):
             os.mkdir(self.output_folder_apk)
 
-        total_packages = len(self.packages)
         counter = 0
-        for package in self.packages:
+        for package in packages_selection:
             counter += 1
 
-            log.info("[%d/%d] Package: %s", counter, total_packages, package.name)
+            log.info("[%d/%d] Package: %s", counter, len(packages_selection),
+                     package["package_name"])
 
+            # Get the file path for the specific package.
             try:
-                output = self._adb_command(f"pm path {package.name}")
-                output = self._clean_output(output)
+                output = self._adb_command(f"pm path {package['package_name']}")
+                output = output.strip().replace("package:", "")
                 if not output:
                     continue
             except Exception as e:
-                log.exception("Failed to get path of package %s: %s", package.name, e)
+                log.exception("Failed to get path of package %s: %s",
+                              package["package_name"], e)
                 self._adb_reconnect()
                 continue
 
@@ -188,33 +166,36 @@ class DownloadAPKs(AndroidExtraction):
             # We loop through each line and download each file.
             for path in output.split("\n"):
                 device_path = path.strip()
-                file_path = self.pull_package_file(package.name, device_path)
+                file_path = self.pull_package_file(package["package_name"],
+                                                   device_path)
                 if not file_path:
                     continue
 
-                # We add the apk metadata to the package object.
-                package.files.append({
+                file_info = {
                     "path": device_path,
                     "local_name": file_path,
                     "sha256": get_sha256_from_file_path(file_path),
-                })
+                }
+
+                if "files" not in package:
+                    package["files"] = [file_info,]
+                else:
+                    package["files"].append(file_info)
+
+        log.info("Download of selected packages completed")
 
     def save_json(self):
         """Save the results to the package.json file.
         """
         json_path = os.path.join(self.output_folder, "apks.json")
-        packages = []
-        for package in self.packages:
-            packages.append(package.__dict__)
-
         with open(json_path, "w") as handle:
-            json.dump(packages, handle, indent=4)
+            json.dump(self.packages, handle, indent=4)
 
     def run(self):
         """Run all steps of fetch-apk.
         """
-        self._adb_connect()
         self.get_packages()
+        self._adb_connect()
         self.pull_packages()
         self.save_json()
         self._adb_disconnect()

mvt/android/lookups/koodous.py

@@ -27,12 +27,12 @@ def koodous_lookup(packages):
     total_packages = len(packages)
     for i in track(range(total_packages), description=f"Looking up {total_packages} packages..."):
         package = packages[i]
-        for file in package.files:
+        for file in package.get("files", []):
             url = f"https://api.koodous.com/apks/{file['sha256']}"
             res = requests.get(url)
             report = res.json()
 
-            row = [package.name, file["local_name"]]
+            row = [package["package_name"], file["local_name"]]
 
             if "package_name" in report:
                 trusted = "no"

mvt/android/lookups/virustotal.py

@@ -41,7 +41,7 @@ def virustotal_lookup(packages):
 
     unique_hashes = []
     for package in packages:
-        for file in package.files:
+        for file in package.get("files", []):
             if file["sha256"] not in unique_hashes:
                 unique_hashes.append(file["sha256"])
 
@@ -74,8 +74,8 @@ def virustotal_lookup(packages):
     table.add_column("Detections")
 
     for package in packages:
-        for file in package.files:
-            row = [package.name, file["local_name"]]
+        for file in package.get("files", []):
+            row = [package["package_name"], file["local_name"]]
 
             if file["sha256"] in detections:
                 detection = detections[file["sha256"]]

mvt/android/modules/adb/packages.py

@@ -75,7 +75,10 @@ class Packages(AndroidExtraction):
                 if installer == "null":
                     installer = None
 
-            uid = fields[2].split(":")[1].strip()
+            try:
+                uid = fields[2].split(":")[1].strip()
+            except IndexError:
+                uid = None
 
             dumpsys = self._adb_command(f"dumpsys package {package_name} | grep -A2 timeStamp").split("\n")
             timestamp = dumpsys[0].split("=")[1].strip()

mvt/common/help.py

@@ -0,0 +1,14 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+# Help messages of repeating options.
+HELP_MSG_OUTPUT = "Specify a path to a folder where you want to store JSON results"
+HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)"
+HELP_MSG_FAST = "Avoid running time/resource consuming features"
+HELP_MSG_LIST_MODULES = "Print list of available modules and exit"
+HELP_MSG_MODULE = "Name of a single module you would like to run instead of all"
+
+# Android-specific.
+HELP_MSG_SERIAL = "Specify a device serial number or HOST:PORT connection string"

mvt/common/module.py

@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import csv
-import glob
 import io
 import os
 import re

mvt/ios/cli.py

@@ -10,6 +10,7 @@ import click
 from rich.logging import RichHandler
 from rich.prompt import Prompt
 
+from mvt.common.help import *
 from mvt.common.indicators import Indicators, IndicatorsFileBadFormat
 from mvt.common.module import run_module, save_timeline
 from mvt.common.options import MutuallyExclusiveOption
@@ -25,9 +26,6 @@ logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
     RichHandler(show_path=False, log_time_format="%X")])
 log = logging.getLogger(__name__)
 
-# Help messages of repeating options.
-OUTPUT_HELP_MESSAGE = "Specify a path to a folder where you want to store JSON results"
-
 # Set this environment variable to a password if needed.
 PASSWD_ENV = "MVT_IOS_BACKUP_PASSWORD"
 
@@ -122,11 +120,11 @@ def extract_key(password, backup_path, key_file):
 #==============================================================================
 @cli.command("check-backup", help="Extract artifacts from an iTunes backup")
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
-              default=[], help="Path to indicators file (can be invoked multiple time)")
-@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE)
-@click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
+              default=[], help=HELP_MSG_IOC)
+@click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT)
+@click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST)
+@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
+@click.option("--module", "-m", help=HELP_MSG_MODULE)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
@@ -185,11 +183,11 @@ def check_backup(ctx, iocs, output, fast, backup_path, list_modules, module):
 #==============================================================================
 @cli.command("check-fs", help="Extract artifacts from a full filesystem dump")
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
-              default=[], help="Path to indicators file (can be invoked multiple time)")
-@click.option("--output", "-o", type=click.Path(exists=False), help=OUTPUT_HELP_MESSAGE)
-@click.option("--fast", "-f", is_flag=True, help="Avoid running time/resource consuming features")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
+              default=[], help=HELP_MSG_IOC)
+@click.option("--output", "-o", type=click.Path(exists=False), help=HELP_MSG_OUTPUT)
+@click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST)
+@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
+@click.option("--module", "-m", help=HELP_MSG_MODULE)
 @click.argument("DUMP_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
@@ -249,9 +247,9 @@ def check_fs(ctx, iocs, output, fast, dump_path, list_modules, module):
 #==============================================================================
 @cli.command("check-iocs", help="Compare stored JSON results to provided indicators")
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
-              default=[], required=True, help="Path to indicators file (can be invoked multiple time)")
-@click.option("--list-modules", "-l", is_flag=True, help="Print list of available modules and exit")
-@click.option("--module", "-m", help="Name of a single module you would like to run instead of all")
+              default=[], required=True, help=HELP_MSG_IOC)
+@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
+@click.option("--module", "-m", help=HELP_MSG_MODULE)
 @click.argument("FOLDER", type=click.Path(exists=True))
 @click.pass_context
 def check_iocs(ctx, iocs, list_modules, module, folder):

mvt/ios/decrypt.py

@@ -8,6 +8,7 @@ import glob
 import logging
 import os
 import shutil
+import sqlite3
 
 from iOSbackup import iOSbackup
 
@@ -30,7 +31,21 @@ class DecryptBackup:
 
     def can_process(self) -> bool:
         return self._backup is not None
-        
+
+    def is_encrypted(self, backup_path) -> bool:
+        """Query Manifest.db file to see if it's encrypted or not.
+        :param backup_path: Path to the backup to decrypt
+        """
+        conn = sqlite3.connect(os.path.join(backup_path, "Manifest.db"))
+        cur = conn.cursor()
+        try:
+            cur.execute("SELECT fileID FROM Files LIMIT 1;")
+        except sqlite3.DatabaseError:
+            return True
+        else:
+            log.critical("The backup does not seem encrypted!")
+            return False
+
     def process_backup(self):
         if not os.path.exists(self.dest_path):
             os.makedirs(self.dest_path)
@@ -94,6 +109,11 @@ class DecryptBackup:
                 log.critical("No Manifest.plist in %s, and %d Manifest.plist files in subdirs. Please choose one!",
                              self.backup_path, len(possible))
                 return
+
+        # Before proceeding, we check whether the backup is indeed encrypted.
+        if not self.is_encrypted(self.backup_path):
+            return
+
         try:
             self._backup = iOSbackup(udid=os.path.basename(self.backup_path),
                                      cleartextpassword=password,
@@ -115,6 +135,10 @@ class DecryptBackup:
         log.info("Decrypting iOS backup at path %s with key file %s",
                  self.backup_path, key_file)
 
+        # Before proceeding, we check whether the backup is indeed encrypted.
+        if not self.is_encrypted(self.backup_path):
+            return
+
         with open(key_file, "rb") as handle:
             key_bytes = handle.read()
 

mvt/ios/modules/backup/configuration_profiles.py

@@ -3,7 +3,6 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import os
 import plistlib
 from base64 import b64encode
 

mvt/ios/modules/backup/manifest.py

@@ -126,7 +126,8 @@ class Manifest(IOSExtraction):
                         "size": self._get_key(file_metadata, "Size"),
                     })
                 except:
-                    self.log.exception("Error reading manifest file metadata")
+                    self.log.exception("Error reading manifest file metadata for file with ID %s and relative path %s",
+                                       file_data["fileID"], file_data["relativePath"])
                     pass
 
             self.results.append(cleaned_metadata)

mvt/ios/modules/backup/profile_events.py

@@ -4,7 +4,6 @@
 #   https://license.mvt.re/1.1/
 
 import plistlib
-from datetime import datetime
 
 from mvt.common.utils import convert_timestamp_to_iso
 

mvt/ios/modules/mixed/webkit_session_resource_log.py

@@ -3,7 +3,6 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import glob
 import os
 import plistlib
 

setup.py

@@ -8,7 +8,7 @@ import os
 from setuptools import find_packages, setup
 
 __package_name__ = "mvt"
-__version__ = "1.2.1"
+__version__ = "1.2.3"
 __description__ = "Mobile Verification Toolkit"
 
 this_directory = os.path.abspath(os.path.dirname(__file__))