Bumps version

* Added Documentation for Encrypted Iphone Backup with Finder on MacOS

* Added details on where to check the backups after completion and added screenshots for the process

* Added location of backups

.github/workflows/flake8.yml

@@ -1,26 +0,0 @@
-name: Flake8
-
-on:
-  push:
-    paths:
-      - '*.py'
-
-jobs:
-  flake8_py3:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.9
-          architecture: x64
-      - name: Checkout
-        uses: actions/checkout@master
-      - name: Install flake8
-        run: pip install flake8
-      - name: Run flake8
-        uses: suo/flake8-github-action@releases/v1
-        with:
-          checkName: 'flake8_py3'   # NOTE: this needs to be the same as the job name
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/python-package.yml

@@ -16,8 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # python-version: [3.7, 3.8, 3.9]
-        python-version: [3.8, 3.9]
+        python-version: ['3.8', '3.9', '3.10']
 
     steps:
     - uses: actions/checkout@v2
@@ -27,6 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
+        python -m pip install --upgrade setuptools
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest safety stix2 pytest-mock
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

.github/workflows/ruff.yml

@@ -0,0 +1,21 @@
+name: Ruff
+on: [push]
+
+jobs:
+  ruff_py3:
+    name: Ruff syntax check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+          architecture: x64
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: Install Dependencies
+        run: |
+          pip install ruff
+      - name: ruff
+        run: |
+          ruff check .

CONTRIBUTING.md

@@ -16,4 +16,4 @@ When contributing code to
 
 - **Quotes**: we use double quotes (`"`) as a default. Single quotes (`'`) can be favored with nested strings instead of escaping (`\"`), or when using f-formatting.
 
-- **Maximum line length**: we strongly encourage to respect a 80 characters long lines and to follow [PEP8 indentation guidelines](https://peps.python.org/pep-0008/#indentation) when having to wrap. However, if breaking at 80 is not possible or is detrimental to the readability of the code, exceptions are tolerated so long as they remain within a hard maximum length of 100 characters.
+- **Maximum line length**: we strongly encourage to respect a 80 characters long lines and to follow [PEP8 indentation guidelines](https://peps.python.org/pep-0008/#indentation) when having to wrap. However, if breaking at 80 is not possible or is detrimental to the readability of the code, exceptions are tolerated. For example, long log lines, or long strings can be extended to 100 characters long. Please hard wrap anything beyond 100 characters.

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # Ref. https://github.com/mvt-project/mvt
 
@@ -7,13 +7,12 @@ LABEL vcs-url="https://github.com/mvt-project/mvt"
 LABEL description="MVT is a forensic tool to look for signs of infection in smartphone devices."
 
 ENV PIP_NO_CACHE_DIR=1
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Fixing major OS dependencies
 # ----------------------------
 RUN apt update \
-  && apt install -y python3 python3-pip libusb-1.0-0-dev \
-  && apt install -y wget unzip\ 
-  && DEBIAN_FRONTEND=noninteractive apt-get -y install default-jre-headless \
+  && apt install -y python3 python3-pip libusb-1.0-0-dev wget unzip default-jre-headless adb \
 
 # Install build tools for libimobiledevice
 # ----------------------------------------
@@ -67,18 +66,9 @@ RUN mkdir /opt/abe \
 # Create alias for abe
   && echo 'alias abe="java -jar /opt/abe/abe.jar"' >> ~/.bashrc
 
-# Install Android Platform Tools 
-# ------------------------------ 
-
-RUN mkdir /opt/android \
-  && wget -q https://dl.google.com/android/repository/platform-tools-latest-linux.zip \
-  && unzip platform-tools-latest-linux.zip -d /opt/android \
-# Create alias for adb 
-  && echo 'alias adb="/opt/android/platform-tools/adb"' >> ~/.bashrc 
-
 # Generate adb key folder 
 # ------------------------------ 
-RUN mkdir /root/.android && /opt/android/platform-tools/adb keygen /root/.android/adbkey
+RUN mkdir /root/.android && adb keygen /root/.android/adbkey
 
 # Setup investigations environment
 # --------------------------------

Makefile

@@ -1,5 +1,10 @@
 PWD = $(shell pwd)
 
+check:
+	flake8
+	pytest -q
+	ruff check -q .
+
 clean:
 	rm -rf $(PWD)/build $(PWD)/dist $(PWD)/mvt.egg-info
 

docs/android/backup.md

@@ -24,7 +24,7 @@ Some recent phones will enforce the utilisation of a password to encrypt the bac
 
 ## Unpack and check the backup
 
-MVT includes a partial implementation of the Android Backup parsing, because of the implementation difference in the compression algorithm between Java and Python. The `-nocompress` option passed to adb in the section above allows to avoid this issue. You can analyse and extract SMSs containing links from the backup directly with MVT:
+MVT includes a partial implementation of the Android Backup parsing, because of the implementation difference in the compression algorithm between Java and Python. The `-nocompress` option passed to adb in the section above allows to avoid this issue. You can analyse and extract SMSs from the backup directly with MVT:
 
 ```bash
 $ mvt-android check-backup --output /path/to/results/ /path/to/backup.ab
@@ -32,7 +32,7 @@ $ mvt-android check-backup --output /path/to/results/ /path/to/backup.ab
          INFO     [mvt.android.modules.backup.sms] Running module SMS...
          INFO     [mvt.android.modules.backup.sms] Processing SMS backup file at
                   apps/com.android.providers.telephony/d_f/000000_sms_backup
-         INFO     [mvt.android.modules.backup.sms] Extracted a total of 64 SMS messages containing links
+         INFO     [mvt.android.modules.backup.sms] Extracted a total of 64 SMS messages
 ```
 
 If the backup is encrypted, MVT will prompt you to enter the password.
@@ -52,4 +52,4 @@ If the backup is encrypted, ABE will prompt you to enter the password.
 
 Alternatively, [ab-decrypt](https://github.com/joernheissler/ab-decrypt) can be used for that purpose.
 
-You can then extract SMSs containing links with MVT by passing the folder path as parameter instead of the `.ab` file: `mvt-android check-backup --output /path/to/results/ /path/to/backup/` (the path to backup given should be the folder containing the `apps` folder).
+You can then extract SMSs with MVT by passing the folder path as parameter instead of the `.ab` file: `mvt-android check-backup --output /path/to/results/ /path/to/backup/` (the path to backup given should be the folder containing the `apps` folder).

docs/docker.md

@@ -1,4 +1,4 @@
-Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed.
+Using Docker simplifies having all the required dependencies and tools (including most recent versions of [libimobiledevice](https://libimobiledevice.org)) readily installed. Note that this requires a Linux host, as Docker for Windows and Mac [doesn't support passing through USB devices](https://docs.docker.com/desktop/faqs/#can-i-pass-through-a-usb-device-to-a-container).
 
 Install Docker following the [official documentation](https://docs.docker.com/get-docker/).
 
@@ -10,11 +10,6 @@ cd mvt
 docker build -t mvt .
 ```
 
-Optionally, you may need to specify your platform to Docker in order to build successfully (Apple M1)
-```bash
-docker build --platform amd64 -t mvt .
-```
-
 Test if the image was created successfully:
 
 ```bash

docs/install.md

@@ -54,7 +54,7 @@ Then you can install MVT directly from [pypi](https://pypi.org/project/mvt/)
 pip3 install mvt
 ```
 
-Or from the source code:
+If you want to have the latest features in development, you can install MVT directly from the source code. If you installed MVT previously from pypi, you should first uninstall it using `pip3 uninstall mvt` and then install from the source code:
 
 ```bash
 git clone https://github.com/mvt-project/mvt.git

docs/ios/backup/itunes.md

@@ -1,16 +1,41 @@
 # Backup with iTunes app
 
-It is possible to do an iPhone backup by using iTunes on Windows or macOS computers (in most recent versions of macOS, this feature is included in Finder).
+It is possible to do an iPhone backup by using iTunes on Windows or macOS computers (in most recent versions of macOS, this feature is included in Finder, see below).
 
 To do that:
 
-* Make sure iTunes is installed.
-* Connect your iPhone to your computer using a Lightning/USB cable.
-* Open the device in iTunes (or Finder on macOS).
-* If you want to have a more accurate detection, ensure that the encrypted backup option is activated and choose a secure password for the backup.
-* Start the backup and wait for it to finish (this may take up to 30 minutes).
+1. Make sure iTunes is installed.
+2. Connect your iPhone to your computer using a Lightning/USB cable.
+3. Open the device in iTunes (or Finder on macOS).
+4. If you want to have a more accurate detection, ensure that the encrypted backup option is activated and choose a secure password for the backup.
+5. Start the backup and wait for it to finish (this may take up to 30 minutes).
 
 ![](../../../img/macos-backup.jpg)
 _Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
 
-* Once the backup is done, find its location and copy it to a place where it can be analyzed by MVT. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On macOS, the backup is stored in `~/Library/Application Support/MobileSync/`.
+Once the backup is done, find its location and copy it to a place where it can be analyzed by MVT. On Windows, the backup can be stored either in `%USERPROFILE%\Apple\MobileSync\` or `%USERPROFILE%\AppData\Roaming\Apple Computer\MobileSync\`. On macOS, the backup is stored in `~/Library/Application Support/MobileSync/`.
+
+# Backup with Finder
+
+On more recent MacOS versions, this feature is included in Finder. To do a backup:
+
+1. Launch Finder on your Mac.
+2. Connect your iPhone to your Mac using a Lightning/USB cable.
+3. Select your device from the list of devices located at the bottom of the left side bar labeled "locations".
+4. In the General tab, select `Back up all the data on your iPhone to this Mac` from the options under the Backups section.
+5. Check the box that says `Encrypt local backup`. If it is your first time selecting this option, you may need to enter a password to encrypt the backup.
+
+![](../../../img/macos-backup2.png)
+_Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
+
+6. Click `Back Up Now` to start the back-up process.
+7. The encrypted backup for your iPhone should now start. Once the process finishes, you can check the backup by opening `Finder`, clicking on the `General` tab, then click on `Manage Backups`. Now you should see a list of your backups like the image below:
+
+![](../../../img/macos-backups.png)
+_Source: [Apple Support](https://support.apple.com/en-us/HT211229)_
+
+If your backup has a lock next to it like in the image above, then the backup is encrypted. You should also see the date and time when the encrypted backup was created. The backup files are stored in `~/Library/Application Support/MobileSync/`.
+
+## Notes:
+
+- Remember to keep the backup encryption password that you created safe, since without it you will not be able to access/modify/decrypt the backup file.

docs/ios/backup/libimobiledevice.md

@@ -3,10 +3,10 @@
 If you have correctly [installed libimobiledevice](../install.md) you can easily generate an iTunes backup using the `idevicebackup2` tool included in the suite. First, you might want to ensure that backup encryption is enabled (**note: encrypted backup contain more data than unencrypted backups**):
 
 ```bash
-idevicebackup2 -i backup encryption on
+idevicebackup2 -i encryption on
 ```
 
-Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 -i backup changepw`, or by turning off encryption (`idevicebackup2 -i backup encryption off`) and turning it back on again.
+Note that if a backup password was previously set on this device, you might need to use the same or change it. You can try changing password using `idevicebackup2 -i changepw`, or by turning off encryption (`idevicebackup2 -i encryption off`) and turning it back on again.
 
 If you are not able to recover or change the password, you should try to disable encryption and obtain an unencrypted backup.
 

docs/ios/records.md

@@ -7,7 +7,7 @@ In this page you can find a (reasonably) up-to-date breakdown of the files creat
 ### `analytics.json`
 
 !!! info "Availability"
-    Backup (if encrypted): :material-close:  
+    Backup (if encrypted): :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Analytics` module. The module extracts records from the plists inside the SQLite databases located at *private/var/Keychains/Analytics/\*.db*, which contain various analytics information regarding networking, certificate-pinning, TLS, etc. failures.
@@ -19,7 +19,7 @@ If indicators are provided through the command-line, processes and domains are c
 ### `backup_info.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-close:
 
 This JSON file is created by mvt-ios' `BackupInfo` module. The module extracts some details about the backup and the device, such as name, phone number, IMEI, product type and version.
@@ -29,7 +29,7 @@ This JSON file is created by mvt-ios' `BackupInfo` module. The module extracts s
 ### `cache_files.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `CacheFiles` module. The module extracts records from all SQLite database files stored on disk with the name *Cache.db*. These databases typically contain data from iOS' [internal URL caching](https://developer.apple.com/documentation/foundation/nsurlcache). Through this module you might be able to recover records of HTTP requests and responses performed my applications as well as system services, that would otherwise be unavailable. For example, you might see HTTP requests part of an exploitation chain performed by an iOS service attempting to download a first stage malicious payload.
@@ -41,7 +41,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `calls.json`
 
 !!! info "Availability"
-    Backup (if encrypted): :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Calls` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CallHistoryDB/CallHistory.storedata*, which contains records of incoming and outgoing calls, including from messaging apps such as WhatsApp or Skype.
@@ -51,7 +51,7 @@ This JSON file is created by mvt-ios' `Calls` module. The module extracts record
 ### `chrome_favicon.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `ChromeFavicon` module. The module extracts records from a SQLite database located at */private/var/mobile/Containers/Data/Application/\*/Library/Application Support/Google/Chrome/Default/Favicons*, which contains a mapping of favicons' URLs and the visited URLs which loaded them.
@@ -63,7 +63,7 @@ If indicators are provided through the command-line, they are checked against bo
 ### `chrome_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `ChromeHistory` module. The module extracts records from a SQLite database located at */private/var/mobile/Containers/Data/Application/\*/Library/Application Support/Google/Chrome/Default/History*, which contains a history of URL visits.
@@ -75,7 +75,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `configuration_profiles.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-close:
 
 This JSON file is created by mvt-ios' `ConfigurationProfiles` module. The module extracts details about iOS configuration profiles that have been installed on the device. These should include both default iOS as well as third-party profiles.
@@ -87,7 +87,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `contacts.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Contacts` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/AddressBook/AddressBook.sqlitedb*, which contains records from the phone's address book. While this database obviously would not contain any malicious indicators per se, you might want to use it to compare records from other apps (such as iMessage, SMS, etc.) to filter those originating from unknown origins.
@@ -97,7 +97,7 @@ This JSON file is created by mvt-ios' `Contacts` module. The module extracts rec
 ### `firefox_favicon.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `FirefoxFavicon` module. The module extracts records from a SQLite database located at */private/var/mobile/profile.profile/browser.db*, which contains a mapping of favicons' URLs and the visited URLs which loaded them.
@@ -109,7 +109,7 @@ If indicators are provided through the command-line, they are checked against bo
 ### `firefox_history.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `FirefoxHistory` module. The module extracts records from a SQLite database located at */private/var/mobile/profile.profile/browser.db*, which contains a history of URL visits.
@@ -121,7 +121,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `id_status_cache.json`
 
 !!! info "Availability"
-    Backup (before iOS 14.7): :material-check:  
+    Backup (before iOS 14.7): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `IDStatusCache` module. The module extracts records from a plist file located at */private/var/mobile/Library/Preferences/com.apple.identityservices.idstatuscache.plist*, which contains a cache of Apple user ID authentication. This chance will indicate when apps like Facetime and iMessage first established contacts with other registered Apple IDs. This is significant because it might contain traces of malicious accounts involved in exploitation of those apps.
@@ -133,7 +133,7 @@ Starting from iOS 14.7.0, this file is empty or absent.
 ### `shortcuts.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Shortcuts` module. The module extracts records from an SQLite database located at */private/var/mobile/Library/Shortcuts/Shortcuts.sqlite*, which contains records about the Shortcuts application. Shortcuts are a built-in iOS feature which allows users to automation certain actions on their device. In some cases the legitimate Shortcuts app may be abused by spyware to maintain persistence on an infected devices.
@@ -143,7 +143,7 @@ This JSON file is created by mvt-ios' `Shortcuts` module. The module extracts re
 ### `interaction_c.json`
 
 !!! info "Availability"
-    Backup (if encrypted): :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `InteractionC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/CoreDuet/People/interactionC.db*, which contains details about user interactions with installed apps.
@@ -153,7 +153,7 @@ This JSON file is created by mvt-ios' `InteractionC` module. The module extracts
 ### `locationd_clients.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `LocationdClients` module. The module extracts records from a plist file located at */private/var/mobile/Library/Caches/locationd/clients.plist*, which contains a cache of apps which requested access to location services.
@@ -163,7 +163,7 @@ This JSON file is created by mvt-ios' `LocationdClients` module. The module extr
 ### `manifest.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-close:
 
 This JSON file is created by mvt-ios' `Manifest` module. The module extracts records from the SQLite database *Manifest.db* contained in iTunes backups, and which indexes the locally backed-up files to the original paths on the iOS device.
@@ -175,7 +175,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `os_analytics_ad_daily.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `OSAnalyticsADDaily` module. The module extracts records from a plist located *private/var/mobile/Library/Preferences/com.apple.osanalytics.addaily.plist*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe.
@@ -187,7 +187,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `datausage.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Datausage` module. The module extracts records from a SQLite database located */private/var/wireless/Library/Databases/DataUsage.sqlite*, which contains a history of network data usage by processes running on the system. It does not log network traffic through WiFi (the fields `WIFI_IN` and `WIFI_OUT` are always empty), and the `WWAN_IN` and `WWAN_OUT` fields are stored in bytes. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention.
@@ -199,7 +199,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `netusage.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `Netusage` module. The module extracts records from a SQLite database located */private/var/networkd/netusage.sqlite*, which contains a history of data usage by processes running on the system. Besides the network statistics, these records are particularly important because they might show traces of malicious process executions and the relevant timeframe. In particular, processes which do not have a valid bundle ID might require particular attention.
@@ -211,7 +211,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `profile_events.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-close:
 
 This JSON file is created by mvt-ios' `ProfileEvents` module. The module extracts a timeline of configuration profile operations. For example, it should indicate when a new profile was installed from the Settings app, or when one was removed.
@@ -221,7 +221,7 @@ This JSON file is created by mvt-ios' `ProfileEvents` module. The module extract
 ### `safari_browser_state.json`
 
 !!! info "Availability"
-    Backup (if encrypted): :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariBrowserState` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/BrowserState.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/BrowserState.db*, which contain records of opened tabs.
@@ -233,7 +233,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `safari_favicon.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariFavicon` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Image Cache/Favicons/Favicons.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Image Cache/Favicons/Favicons.db*, which contain mappings of favicons' URLs and the visited URLs which loaded them.
@@ -245,7 +245,7 @@ If indicators are provided through the command-line, they are checked against bo
 ### `safari_history.json`
 
 !!! info "Availability"
-    Backup (if encrypted): :material-check:  
+    Backup (if encrypted): :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SafariHistory` module. The module extracts records from the SQLite databases located at */private/var/mobile/Library/Safari/History.db* or */private/var/mobile/Containers/Data/Application/\*/Library/Safari/History.db*, which contain a history of URL visits.
@@ -257,7 +257,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `shutdown_log.json`
 
 !!! info "Availability"
-    Backup (if encrypted): :material-close:  
+    Backup (if encrypted): :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `ShutdownLog` module. The module extracts records from the shutdown log located at *private/var/db/diagnostics/shutdown.log*. When shutting down an iPhone, a SIGTERM will be sent to all processes runnning. The `shutdown.log` file will log any process (with its pid and path) that did not shut down after the SIGTERM was sent.
@@ -269,10 +269,10 @@ If indicators are provided through the command-line, they are checked against th
 ### `sms.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `SMS` module. The module extracts a list of SMS messages containing HTTP links from the SQLite database located at */private/var/mobile/Library/SMS/sms.db*.
+This JSON file is created by mvt-ios' `SMS` module. The module extracts a list of SMS messages from the SQLite database located at */private/var/mobile/Library/SMS/sms.db*.
 
 If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *sms_detected.json*.
 
@@ -281,7 +281,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `sms_attachments.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `SMSAttachments` module. The module extracts details about attachments sent via SMS or iMessage from the same database used by the `SMS` module. These records might be useful to indicate unique patterns that might be indicative of exploitation attempts leveraging potential vulnerabilities in file format parsers or other forms of file handling by the Messages app.
@@ -291,7 +291,7 @@ This JSON file is created by mvt-ios' `SMSAttachments` module. The module extrac
 ### `tcc.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `TCC` module. The module extracts records from a SQLite database located at */private/var/mobile/Library/TCC/TCC.db*, which contains a list of which services such as microphone, camera, or location, apps have been granted or denied access to.
@@ -301,7 +301,7 @@ This JSON file is created by mvt-ios' `TCC` module. The module extracts records
 ### `version_history.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `IOSVersionHistory` module. The module extracts records of iOS software updates from analytics plist files located at */private/var/db/analyticsd/Analytics-Journal-\*.ips*.
@@ -311,7 +311,7 @@ This JSON file is created by mvt-ios' `IOSVersionHistory` module. The module ext
 ### `webkit_indexeddb.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitIndexedDB` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/IndexedDB*, which contains IndexedDB files created by any app installed on the device.
@@ -323,7 +323,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `webkit_local_storage.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitLocalStorage` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/Library/WebKit/WebsiteData/LocalStorage/*, which contains local storage files created by any app installed on the device.
@@ -335,7 +335,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `webkit_resource_load_statistics.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios `WebkitResourceLoadStatistics` module. The module extracts records from available WebKit ResourceLoadStatistics *observations.db* SQLite3 databases. These records should indicate domain names contacted by apps, including a timestamp.
@@ -347,7 +347,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `webkit_safari_view_service.json`
 
 !!! info "Availability"
-    Backup: :material-close:  
+    Backup: :material-close:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitSafariViewService` module. The module extracts a list of file and folder names located at the following path */private/var/mobile/Containers/Data/Application/\*/SystemData/com.apple.SafariViewService/Library/WebKit/WebsiteData/*, which contains files cached by SafariVewService.
@@ -359,7 +359,7 @@ If indicators are provided through the command-line, they are checked against th
 ### `webkit_session_resource_log.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
 This JSON file is created by mvt-ios' `WebkitSessionResourceLog` module. The module extracts records from plist files with the name *full_browsing_session_resourceLog.plist*, which contain records of resources loaded by different domains visited.
@@ -371,10 +371,10 @@ If indicators are provided through the command-line, they are checked against th
 ### `whatsapp.json`
 
 !!! info "Availability"
-    Backup: :material-check:  
+    Backup: :material-check:
     Full filesystem dump: :material-check:
 
-This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages containing HTTP links from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*.
+This JSON file is created by mvt-ios' `WhatsApp` module. The module extracts a list of WhatsApp messages from the SQLite database located at *private/var/mobile/Containers/Shared/AppGroup/\*/ChatStorage.sqlite*.
 
 If indicators are provided through the command-line, they are checked against the extracted HTTP links. Any matches are stored in *whatsapp_detected.json*.
 

mvt/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/cli.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -9,13 +9,14 @@ import click
 from rich.logging import RichHandler
 
 from mvt.common.cmd_check_iocs import CmdCheckIOCS
-from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_HASHES, HELP_MSG_IOC,
                              HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
                              HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
 from mvt.common.logo import logo
 from mvt.common.updates import IndicatorsUpdates
 
 from .cmd_check_adb import CmdAndroidCheckADB
+from .cmd_check_androidqf import CmdAndroidCheckAndroidQF
 from .cmd_check_backup import CmdAndroidCheckBackup
 from .cmd_check_bugreport import CmdAndroidCheckBugreport
 from .cmd_download_apks import DownloadAPKs
@@ -29,6 +30,7 @@ LOG_FORMAT = "[%(name)s] %(message)s"
 logging.basicConfig(level="INFO", format=LOG_FORMAT, handlers=[
     RichHandler(show_path=False, log_time_format="%X")])
 log = logging.getLogger(__name__)
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 
 #==============================================================================
@@ -50,7 +52,8 @@ def version():
 #==============================================================================
 # Command: download-apks
 #==============================================================================
-@cli.command("download-apks", help="Download all or only non-system installed APKs")
+@cli.command("download-apks", help="Download all or only non-system installed APKs",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
 @click.option("--all-apks", "-a", is_flag=True,
               help="Extract all packages installed on the phone, including system packages")
@@ -99,7 +102,8 @@ def download_apks(ctx, all_apks, virustotal, output, from_file, serial):
 #==============================================================================
 # Command: check-adb
 #==============================================================================
-@cli.command("check-adb", help="Check an Android device over adb")
+@cli.command("check-adb", help="Check an Android device over adb",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--serial", "-s", type=str, help=HELP_MSG_SERIAL)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
@@ -121,15 +125,16 @@ def check_adb(ctx, serial, iocs, output, fast, list_modules, module):
 
     cmd.run()
 
-    if len(cmd.timeline_detected) > 0:
+    if cmd.detected_count > 0:
         log.warning("The analysis of the Android device produced %d detections!",
-                    len(cmd.timeline_detected))
+                    cmd.detected_count)
 
 
 #==============================================================================
 # Command: check-bugreport
 #==============================================================================
-@cli.command("check-bugreport", help="Check an Android Bug Report")
+@cli.command("check-bugreport", help="Check an Android Bug Report",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
 @click.option("--output", "-o", type=click.Path(exists=False),
@@ -139,9 +144,10 @@ def check_adb(ctx, serial, iocs, output, fast, list_modules, module):
 @click.argument("BUGREPORT_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_bugreport(ctx, iocs, output, list_modules, module, bugreport_path):
+    # Always generate hashes as bug reports are small.
     cmd = CmdAndroidCheckBugreport(target_path=bugreport_path,
                                    results_path=output, ioc_files=iocs,
-                                   module_name=module)
+                                   module_name=module, hashes=True)
 
     if list_modules:
         cmd.list_modules()
@@ -151,15 +157,16 @@ def check_bugreport(ctx, iocs, output, list_modules, module, bugreport_path):
 
     cmd.run()
 
-    if len(cmd.timeline_detected) > 0:
+    if cmd.detected_count > 0:
         log.warning("The analysis of the Android bug report produced %d detections!",
-                    len(cmd.timeline_detected))
+                    cmd.detected_count)
 
 
 #==============================================================================
 # Command: check-backup
 #==============================================================================
-@cli.command("check-backup", help="Check an Android Backup")
+@cli.command("check-backup", help="Check an Android Backup",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
 @click.option("--output", "-o", type=click.Path(exists=False),
@@ -168,8 +175,9 @@ def check_bugreport(ctx, iocs, output, list_modules, module, bugreport_path):
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_backup(ctx, iocs, output, list_modules, backup_path):
+    # Always generate hashes as backups are generally small.
     cmd = CmdAndroidCheckBackup(target_path=backup_path, results_path=output,
-                                ioc_files=iocs)
+                                ioc_files=iocs, hashes=True)
 
     if list_modules:
         cmd.list_modules()
@@ -179,15 +187,48 @@ def check_backup(ctx, iocs, output, list_modules, backup_path):
 
     cmd.run()
 
-    if len(cmd.timeline_detected) > 0:
+    if cmd.detected_count > 0:
         log.warning("The analysis of the Android backup produced %d detections!",
-                    len(cmd.timeline_detected))
+                    cmd.detected_count)
+
+
+#==============================================================================
+# Command: check-androidqf
+#==============================================================================
+@cli.command("check-androidqf", help="Check data collected with AndroidQF",
+             context_settings=CONTEXT_SETTINGS)
+@click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
+              default=[], help=HELP_MSG_IOC)
+@click.option("--output", "-o", type=click.Path(exists=False),
+              help=HELP_MSG_OUTPUT)
+@click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
+@click.option("--module", "-m", help=HELP_MSG_MODULE)
+@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
+@click.argument("ANDROIDQF_PATH", type=click.Path(exists=True))
+@click.pass_context
+def check_androidqf(ctx, iocs, output, list_modules, module, hashes, androidqf_path):
+    cmd = CmdAndroidCheckAndroidQF(target_path=androidqf_path,
+                                   results_path=output, ioc_files=iocs,
+                                   module_name=module, hashes=hashes)
+
+    if list_modules:
+        cmd.list_modules()
+        return
+
+    log.info("Checking AndroidQF acquisition at path: %s", androidqf_path)
+
+    cmd.run()
+
+    if cmd.detected_count > 0:
+        log.warning("The analysis of the AndroidQF acquisition produced %d detections!",
+                    cmd.detected_count)
 
 
 #==============================================================================
 # Command: check-iocs
 #==============================================================================
-@cli.command("check-iocs", help="Compare stored JSON results to provided indicators")
+@cli.command("check-iocs", help="Compare stored JSON results to provided indicators",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
 @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
@@ -208,7 +249,8 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
 #==============================================================================
 # Command: download-iocs
 #==============================================================================
-@cli.command("download-iocs", help="Download public STIX2 indicators")
+@cli.command("download-iocs", help="Download public STIX2 indicators",
+             context_settings=CONTEXT_SETTINGS)
 def download_indicators():
     ioc_updates = IndicatorsUpdates()
     ioc_updates.update()

mvt/android/cmd_check_adb.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.common.command import Command
 
@@ -14,9 +15,15 @@ log = logging.getLogger(__name__)
 
 class CmdAndroidCheckADB(Command):
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+    ) -> None:
         super().__init__(target_path=target_path, results_path=results_path,
                          ioc_files=ioc_files, module_name=module_name,
                          serial=serial, fast_mode=fast_mode, log=log)

mvt/android/cmd_check_androidqf.py

@@ -0,0 +1,34 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.common.command import Command
+
+from .modules.androidqf import ANDROIDQF_MODULES
+
+log = logging.getLogger(__name__)
+
+
+class CmdAndroidCheckAndroidQF(Command):
+
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        hashes: Optional[bool] = False,
+    ) -> None:
+        super().__init__(target_path=target_path, results_path=results_path,
+                         ioc_files=ioc_files, module_name=module_name,
+                         serial=serial, fast_mode=fast_mode, hashes=hashes,
+                         log=log)
+
+        self.name = "check-androidqf"
+        self.modules = ANDROIDQF_MODULES

mvt/android/cmd_check_backup.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -9,10 +9,11 @@ import os
 import sys
 import tarfile
 from pathlib import Path
-from typing import Callable
+from typing import List, Optional
 
 from rich.prompt import Prompt
 
+from mvt.android.modules.backup.base import BackupExtraction
 from mvt.android.parsers.backup import (AndroidBackupParsingError,
                                         InvalidBackupPassword, parse_ab_header,
                                         parse_backup_file)
@@ -25,21 +26,32 @@ log = logging.getLogger(__name__)
 
 class CmdAndroidCheckBackup(Command):
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        hashes: Optional[bool] = False,
+    ) -> None:
         super().__init__(target_path=target_path, results_path=results_path,
                          ioc_files=ioc_files, module_name=module_name,
-                         serial=serial, fast_mode=fast_mode, log=log)
+                         serial=serial, fast_mode=fast_mode, hashes=hashes,
+                         log=log)
 
         self.name = "check-backup"
         self.modules = BACKUP_MODULES
 
-        self.backup_type = None
-        self.backup_archive = None
-        self.backup_files = []
+        self.backup_type: str = ""
+        self.backup_archive: Optional[tarfile.TarFile] = None
+        self.backup_files: List[str] = []
 
     def init(self) -> None:
+        if not self.target_path:
+            return
+
         if os.path.isfile(self.target_path):
             self.backup_type = "ab"
             with open(self.target_path, "rb") as handle:
@@ -80,7 +92,7 @@ class CmdAndroidCheckBackup(Command):
                          "Android Backup (.ab) file")
             sys.exit(1)
 
-    def module_init(self, module: Callable) -> None:
+    def module_init(self, module: BackupExtraction) -> None:  # type: ignore[override]
         if self.backup_type == "folder":
             module.from_folder(self.target_path, self.backup_files)
         else:

mvt/android/cmd_check_bugreport.py

@@ -1,14 +1,15 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
 from pathlib import Path
-from typing import Callable
+from typing import List, Optional
 from zipfile import ZipFile
 
+from mvt.android.modules.bugreport.base import BugReportModule
 from mvt.common.command import Command
 
 from .modules.bugreport import BUGREPORT_MODULES
@@ -18,21 +19,32 @@ log = logging.getLogger(__name__)
 
 class CmdAndroidCheckBugreport(Command):
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        hashes: Optional[bool] = False,
+    ) -> None:
         super().__init__(target_path=target_path, results_path=results_path,
                          ioc_files=ioc_files, module_name=module_name,
-                         serial=serial, fast_mode=fast_mode, log=log)
+                         serial=serial, fast_mode=fast_mode, hashes=hashes,
+                         log=log)
 
         self.name = "check-bugreport"
         self.modules = BUGREPORT_MODULES
 
-        self.bugreport_format = None
-        self.bugreport_archive = None
-        self.bugreport_files = []
+        self.bugreport_format: str = ""
+        self.bugreport_archive: Optional[ZipFile] = None
+        self.bugreport_files: List[str] = []
 
     def init(self) -> None:
+        if not self.target_path:
+            return
+
         if os.path.isfile(self.target_path):
             self.bugreport_format = "zip"
             self.bugreport_archive = ZipFile(self.target_path)
@@ -47,8 +59,12 @@ class CmdAndroidCheckBugreport(Command):
                                                 parent_path)
                     self.bugreport_files.append(file_path)
 
-    def module_init(self, module: Callable) -> None:
+    def module_init(self, module: BugReportModule) -> None:  # type: ignore[override]
         if self.bugreport_format == "zip":
             module.from_zip(self.bugreport_archive, self.bugreport_files)
         else:
             module.from_folder(self.target_path, self.bugreport_files)
+
+    def finish(self) -> None:
+        if self.bugreport_archive:
+            self.bugreport_archive.close()

mvt/android/cmd_download_apks.py

@@ -1,12 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import json
 import logging
 import os
-from typing import Callable
+from typing import Callable, Optional
 
 from rich.progress import track
 
@@ -25,8 +25,12 @@ class DownloadAPKs(AndroidExtraction):
 
     """
 
-    def __init__(self, results_path: str = "", all_apks: bool = False,
-                 packages: list = []):
+    def __init__(
+        self,
+        results_path: Optional[str] = None,
+        all_apks: Optional[bool] = False,
+        packages: Optional[list] = None
+    ) -> None:
         """Initialize module.
         :param results_path: Path to the folder where data should be stored
         :param all_apks: Boolean indicating whether to download all packages
@@ -78,9 +82,8 @@ class DownloadAPKs(AndroidExtraction):
         try:
             self._adb_download(remote_path, local_path)
         except InsufficientPrivileges:
-            log.error("Unable to pull package file from %s: insufficient "
-                      "privileges, it might be a system app",
-                      remote_path)
+            log.error("Unable to pull package file from %s: insufficient privileges, "
+                      "it might be a system app", remote_path)
             self._adb_reconnect()
             return None
         except Exception as exc:
@@ -122,8 +125,8 @@ class DownloadAPKs(AndroidExtraction):
                 if not package.get("system", False):
                     packages_selection.append(package)
 
-            log.info("Selected only %d packages which are not marked as "
-                     "\"system\"", len(packages_selection))
+            log.info("Selected only %d packages which are not marked as \"system\"",
+                     len(packages_selection))
 
         if len(packages_selection) == 0:
             log.info("No packages were selected for download")

mvt/android/modules/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/android/modules/adb/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/modules/adb/base.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -11,7 +11,7 @@ import string
 import sys
 import tempfile
 import time
-from typing import Callable
+from typing import Callable, Optional
 
 from adb_shell.adb_device import AdbDeviceTcp, AdbDeviceUsb
 from adb_shell.auth.keygen import keygen, write_public_keyfile
@@ -32,10 +32,15 @@ ADB_PUB_KEY_PATH = os.path.expanduser("~/.android/adbkey.pub")
 class AndroidExtraction(MVTModule):
     """This class provides a base for all Android extraction modules."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -73,15 +78,13 @@ class AndroidExtraction(MVTModule):
             try:
                 self.device = AdbDeviceUsb(serial=self.serial)
             except UsbDeviceNotFoundError:
-                self.log.critical("No device found. Make sure it is connected "
-                                  "and unlocked.")
+                self.log.critical("No device found. Make sure it is connected and unlocked.")
                 sys.exit(-1)
         # Otherwise we try to use the TCP transport.
         else:
             addr = self.serial.split(":")
             if len(addr) < 2:
-                raise ValueError("TCP serial number must follow the format: "
-                                 "`address:port`")
+                raise ValueError("TCP serial number must follow the format: `address:port`")
 
             self.device = AdbDeviceTcp(addr[0], int(addr[1]),
                                        default_transport_timeout_s=30.)
@@ -90,12 +93,11 @@ class AndroidExtraction(MVTModule):
             try:
                 self.device.connect(rsa_keys=[signer], auth_timeout_s=5)
             except (USBErrorBusy, USBErrorAccess):
-                self.log.critical("Device is busy, maybe run `adb kill-server` "
-                                  "and try again.")
+                self.log.critical("Device is busy, maybe run `adb kill-server` and try again.")
                 sys.exit(-1)
             except DeviceAuthError:
-                self.log.error("You need to authorize this computer on the "
-                               "Android device. Retrying in 5 seconds...")
+                self.log.error("You need to authorize this computer on the Android device. "
+                               "Retrying in 5 seconds...")
                 time.sleep(5)
             except UsbReadFailedError:
                 self.log.error("Unable to connect to the device over USB. "
@@ -104,7 +106,7 @@ class AndroidExtraction(MVTModule):
             except OSError as exc:
                 if exc.errno == 113 and self.serial:
                     self.log.critical("Unable to connect to the device %s: "
-                                      "did you specify the correct IP addres?",
+                                      "did you specify the correct IP address?",
                                       self.serial)
                     sys.exit(-1)
             else:
@@ -136,7 +138,8 @@ class AndroidExtraction(MVTModule):
         :returns: Boolean indicating whether a `su` binary is present or not
 
         """
-        return bool(self._adb_command("command -v su"))
+        result = self._adb_command("command -v su && su -c true")
+        return bool(result) and "Permission denied" not in result
 
     def _adb_root_or_die(self) -> None:
         """Check if we have a `su` binary, otherwise raise an Exception."""
@@ -169,9 +172,13 @@ class AndroidExtraction(MVTModule):
 
         return bool(self._adb_command_as_root(f"[ ! -f {file} ] || echo 1"))
 
-    def _adb_download(self, remote_path: str, local_path: str,
-                      progress_callback: Callable = None,
-                      retry_root: bool = True) -> None:
+    def _adb_download(
+        self,
+        remote_path: str,
+        local_path: str,
+        progress_callback: Optional[Callable] = None,
+        retry_root: Optional[bool] = True
+    ) -> None:
         """Download a file form the device.
 
         :param remote_path: Path to download from the device
@@ -190,8 +197,12 @@ class AndroidExtraction(MVTModule):
             else:
                 raise Exception(f"Unable to download file {remote_path}: {exc}") from exc
 
-    def _adb_download_root(self, remote_path: str, local_path: str,
-                           progress_callback: Callable = None) -> None:
+    def _adb_download_root(
+        self,
+        remote_path: str,
+        local_path: str,
+        progress_callback: Optional[Callable] = None
+    ) -> None:
         try:
             # Check if we have root, if not raise an Exception.
             self._adb_root_or_die()
@@ -262,8 +273,8 @@ class AndroidExtraction(MVTModule):
         self._adb_command(f"rm -f {new_remote_path}")
 
     def _generate_backup(self, package_name: str) -> bytes:
-        self.log.warning("Please check phone and accept Android backup prompt. "
-                         "You may need to set a backup password. \a")
+        self.log.info("Please check phone and accept Android backup prompt. "
+                      "You may need to set a backup password. \a")
 
         # TODO: Base64 encoding as temporary fix to avoid byte-mangling over
         #       the shell transport...
@@ -288,10 +299,9 @@ class AndroidExtraction(MVTModule):
                                                          backup_password)
                 return decrypted_backup_tar
             except InvalidBackupPassword:
-                self.log.error("You provided the wrong password! "
-                               "Please try again...")
+                self.log.error("You provided the wrong password! Please try again...")
 
-        self.log.warn("All attempts to decrypt backup with password failed!")
+        self.log.error("All attempts to decrypt backup with password failed!")
 
         return None
 

mvt/android/modules/adb/chrome_history.py

@@ -1,12 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
 import sqlite3
-from typing import Union
+from typing import Optional, Union
 
 from mvt.common.utils import (convert_chrometime_to_datetime,
                               convert_datetime_to_iso)
@@ -19,13 +19,19 @@ CHROME_HISTORY_PATH = "data/data/com.android.chrome/app_chrome/Default/History"
 class ChromeHistory(AndroidExtraction):
     """This module extracts records from Android's Chrome browsing history."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
+        self.results = []
 
     def serialize(self, record: dict) -> Union[dict, list]:
         return {
@@ -50,6 +56,7 @@ class ChromeHistory(AndroidExtraction):
         :param db_path: Path to the History database to process.
 
         """
+        assert isinstance(self.results, list)  # assert results type for mypy
         conn = sqlite3.connect(db_path)
         cur = conn.cursor()
         cur.execute("""
@@ -70,7 +77,8 @@ class ChromeHistory(AndroidExtraction):
                 "url": item[1],
                 "visit_id": item[2],
                 "timestamp": item[3],
-                "isodate": convert_datetime_to_iso(convert_chrometime_to_datetime(item[3])),
+                "isodate": convert_datetime_to_iso(
+                    convert_chrometime_to_datetime(item[3])),
                 "redirect_source": item[4],
             })
 

mvt/android/modules/adb/dumpsys_accessibility.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_accessibility
 
@@ -13,10 +14,15 @@ from .base import AndroidExtraction
 class DumpsysAccessibility(AndroidExtraction):
     """This module extracts stats on accessibility."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/dumpsys_activities.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_activity_resolver_table
 
@@ -13,10 +14,15 @@ from .base import AndroidExtraction
 class DumpsysActivities(AndroidExtraction):
     """This module extracts details on receivers for risky activities."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/dumpsys_appops.py

@@ -1,10 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-from typing import Union
+from typing import Optional, Union
 
 from mvt.android.parsers.dumpsys import parse_dumpsys_appops
 
@@ -16,10 +16,15 @@ class DumpsysAppOps(AndroidExtraction):
 
     slug = "dumpsys_appops"
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/dumpsys_battery_daily.py

@@ -1,10 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-from typing import Union
+from typing import Optional, Union
 
 from mvt.android.parsers import parse_dumpsys_battery_daily
 
@@ -14,10 +14,15 @@ from .base import AndroidExtraction
 class DumpsysBatteryDaily(AndroidExtraction):
     """This module extracts records from battery daily updates."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -49,4 +54,5 @@ class DumpsysBatteryDaily(AndroidExtraction):
 
         self.results = parse_dumpsys_battery_daily(output)
 
-        self.log.info("Extracted %d records from battery daily stats", len(self.results))
+        self.log.info("Extracted %d records from battery daily stats",
+                      len(self.results))

mvt/android/modules/adb/dumpsys_battery_history.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_battery_history
 
@@ -13,10 +14,15 @@ from .base import AndroidExtraction
 class DumpsysBatteryHistory(AndroidExtraction):
     """This module extracts records from battery history events."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/dumpsys_dbinfo.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_dbinfo
 
@@ -15,10 +16,15 @@ class DumpsysDBInfo(AndroidExtraction):
 
     slug = "dumpsys_dbinfo"
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/dumpsys_full.py

@@ -1,10 +1,11 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
+from typing import Optional
 
 from .base import AndroidExtraction
 
@@ -12,10 +13,15 @@ from .base import AndroidExtraction
 class DumpsysFull(AndroidExtraction):
     """This module extracts stats on battery consumption by processes."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/dumpsys_receivers.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
 
@@ -19,10 +20,15 @@ INTENT_NEW_OUTGOING_CALL = "android.intent.action.NEW_OUTGOING_CALL"
 class DumpsysReceivers(AndroidExtraction):
     """This module extracts details on receivers for risky activities."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -36,24 +42,20 @@ class DumpsysReceivers(AndroidExtraction):
         for intent, receivers in self.results.items():
             for receiver in receivers:
                 if intent == INTENT_NEW_OUTGOING_SMS:
-                    self.log.info("Found a receiver to intercept "
-                                  "outgoing SMS messages: \"%s\"",
+                    self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_SMS_RECEIVED:
-                    self.log.info("Found a receiver to intercept "
-                                  "incoming SMS messages: \"%s\"",
+                    self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_DATA_SMS_RECEIVED:
-                    self.log.info("Found a receiver to intercept "
-                                  "incoming data SMS message: \"%s\"",
+                    self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_PHONE_STATE:
                     self.log.info("Found a receiver monitoring "
                                   "telephony state/incoming calls: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_NEW_OUTGOING_CALL:
-                    self.log.info("Found a receiver monitoring "
-                                  "outgoing calls: \"%s\"",
+                    self.log.info("Found a receiver monitoring outgoing calls: \"%s\"",
                                   receiver["receiver"])
 
                 ioc = self.indicators.check_app_id(receiver["package_name"])

mvt/android/modules/adb/files.py

@@ -1,12 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
 import stat
-from typing import Union
+from typing import Optional, Union
 
 from mvt.common.utils import convert_unix_to_iso
 
@@ -25,16 +25,21 @@ ANDROID_MEDIA_FOLDERS = [
 class Files(AndroidExtraction):
     """This module extracts the list of files on the device."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
         self.full_find = False
 
-    def serialize(self, record: dict) -> Union[dict, list]:
+    def serialize(self, record: dict) -> Union[dict, list, None]:
         if "modified_time" in record:
             return {
                 "timestamp": record["modified_time"],
@@ -48,8 +53,8 @@ class Files(AndroidExtraction):
     def check_indicators(self) -> None:
         for result in self.results:
             if result.get("is_suid"):
-                self.log.warning("Found an SUID file in a non-standard "
-                                 "directory \"%s\".", result["path"])
+                self.log.warning("Found an SUID file in a non-standard directory \"%s\".",
+                                 result["path"])
 
             if self.indicators and self.indicators.check_file_path(result["path"]):
                 self.log.warning("Found a known suspicous file at path: \"%s\"",
@@ -57,6 +62,9 @@ class Files(AndroidExtraction):
                 self.detected.append(result)
 
     def backup_file(self, file_path: str) -> None:
+        if not self.results_path:
+            return
+
         local_file_name = file_path.replace("/", "_").replace(" ", "-")
         local_files_folder = os.path.join(self.results_path, "files")
         if not os.path.exists(local_files_folder):
@@ -74,13 +82,18 @@ class Files(AndroidExtraction):
                           file_path, local_file_path)
 
     def find_files(self, folder: str) -> None:
+        assert isinstance(self.results, list)
         if self.full_find:
             cmd = f"find '{folder}' -type f -printf '%T@ %m %s %u %g %p\n' 2> /dev/null"
             output = self._adb_command(cmd)
 
             for file_line in output.splitlines():
+                file_info = file_line.rstrip().split(" ", 5)
+                if len(file_line) < 6:
+                    self.log.info("Skipping invalid file info - %s", file_line.rstrip())
+                    continue
                 [unix_timestamp, mode, size,
-                 owner, group, full_path] = file_line.rstrip().split(" ", 5)
+                 owner, group, full_path] = file_info
                 mod_time = convert_unix_to_iso(unix_timestamp)
 
                 self.results.append({
@@ -112,8 +125,7 @@ class Files(AndroidExtraction):
         for entry in self.results:
             self.log.info("Found file in tmp folder at path %s",
                           entry.get("path"))
-            if self.results_path:
-                self.backup_file(entry.get("path"))
+            self.backup_file(entry.get("path"))
 
         for media_folder in ANDROID_MEDIA_FOLDERS:
             self.find_files(media_folder)
@@ -124,8 +136,7 @@ class Files(AndroidExtraction):
         if self.fast_mode:
             self.log.info("Flag --fast was enabled: skipping full file listing")
         else:
-            self.log.info("Processing full file listing. "
-                          "This may take a while...")
+            self.log.info("Processing full file listing. This may take a while...")
             self.find_files("/")
 
             self.log.info("Found %s total files", len(self.results))

mvt/android/modules/adb/getprop.py

@@ -1,10 +1,11 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 from datetime import datetime, timedelta
+from typing import Optional
 
 from mvt.android.parsers import parse_getprop
 
@@ -14,16 +15,31 @@ from .base import AndroidExtraction
 class Getprop(AndroidExtraction):
     """This module extracts device properties from getprop command."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
 
         self.results = {} if not results else results
 
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_android_property_name(result.get("name", ""))
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+
     def run(self) -> None:
         self._adb_connect()
         output = self._adb_command("getprop")
@@ -32,13 +48,14 @@ class Getprop(AndroidExtraction):
         self.results = parse_getprop(output)
 
         # Alert if phone is outdated.
-        security_patch = self.results.get("ro.build.version.security_patch", "")
-        if security_patch:
-            patch_date = datetime.strptime(security_patch, "%Y-%m-%d")
+        for entry in self.results:
+            if entry.get("name", "") != "ro.build.version.security_patch":
+                continue
+            patch_date = datetime.strptime(entry["value"], "%Y-%m-%d")
             if (datetime.now() - patch_date) > timedelta(days=6*30):
                 self.log.warning("This phone has not received security updates "
                                  "for more than six months (last update: %s)",
-                                 security_patch)
+                                 entry["value"])
 
         self.log.info("Extracted %d Android system properties",
                       len(self.results))

mvt/android/modules/adb/logcat.py

@@ -1,10 +1,11 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
+from typing import Optional
 
 from .base import AndroidExtraction
 
@@ -12,10 +13,15 @@ from .base import AndroidExtraction
 class Logcat(AndroidExtraction):
     """This module extracts details on installed packages."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -24,9 +30,9 @@ class Logcat(AndroidExtraction):
         self._adb_connect()
 
         # Get the current logcat.
-        output = self._adb_command("logcat -d")
+        output = self._adb_command("logcat -d -b all \"*:V\"")
         # Get the locat prior to last reboot.
-        last_output = self._adb_command("logcat -L")
+        last_output = self._adb_command("logcat -L -b all \"*:V\"")
 
         if self.results_path:
             logcat_path = os.path.join(self.results_path,

mvt/android/modules/adb/packages.py

@@ -1,16 +1,17 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-from typing import Union
+from typing import List, Optional, Union
 
 from rich.console import Console
 from rich.progress import track
 from rich.table import Table
 from rich.text import Text
 
+from mvt.android.parsers.dumpsys import parse_dumpsys_package_for_details
 from mvt.common.virustotal import VTNoKey, VTQuotaExceeded, virustotal_lookup
 
 from .base import AndroidExtraction
@@ -38,8 +39,7 @@ DANGEROUS_PERMISSIONS = [
     "android.permission.USE_SIP",
     "com.android.browser.permission.READ_HISTORY_BOOKMARKS",
 ]
-
-ROOT_PACKAGES = [
+ROOT_PACKAGES: List[str] = [
     "com.noshufou.android.su",
     "com.noshufou.android.su.elite",
     "eu.chainfire.supersu",
@@ -66,15 +66,37 @@ ROOT_PACKAGES = [
     "com.kingouser.com",
     "com.topjohnwu.magisk",
 ]
+SECURITY_PACKAGES = [
+    "com.policydm",
+    "com.samsung.android.app.omcagent",
+    "com.samsung.android.securitylogagent",
+    "com.sec.android.soagent",
+]
+SYSTEM_UPDATE_PACKAGES = [
+    "com.android.updater",
+    "com.google.android.gms",
+    "com.huawei.android.hwouc",
+    "com.lge.lgdmsclient",
+    "com.motorola.ccc.ota",
+    "com.oneplus.opbackup",
+    "com.oppo.ota",
+    "com.transsion.systemupdate",
+    "com.wssyncmldm",
+]
 
 
 class Packages(AndroidExtraction):
     """This module extracts the list of installed packages."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -117,6 +139,14 @@ class Packages(AndroidExtraction):
                 self.detected.append(result)
                 continue
 
+            if result["package_name"] in SECURITY_PACKAGES and result["disabled"]:
+                self.log.warning("Found a security package disabled: \"%s\"",
+                                 result["package_name"])
+
+            if result["package_name"] in SYSTEM_UPDATE_PACKAGES and result["disabled"]:
+                self.log.warning("System OTA update package \"%s\" disabled on the phone",
+                                 result["package_name"])
+
             if not self.indicators:
                 continue
 
@@ -187,43 +217,17 @@ class Packages(AndroidExtraction):
 
     @staticmethod
     def parse_package_for_details(output: str) -> dict:
-        details = {
-            "uid": "",
-            "version_name": "",
-            "version_code": "",
-            "timestamp": "",
-            "first_install_time": "",
-            "last_update_time": "",
-            "requested_permissions": [],
-        }
-
-        in_permissions = False
+        lines = []
+        in_packages = False
         for line in output.splitlines():
-            if in_permissions:
-                if line.startswith(" " * 4) and not line.startswith(" " * 6):
-                    in_permissions = False
-                    continue
+            if in_packages:
+                if line.strip() == "":
+                    break
+                lines.append(line)
+            if line.strip() == "Packages:":
+                in_packages = True
 
-                permission = line.strip().split(":")[0]
-                details["requested_permissions"].append(permission)
-
-            if line.strip().startswith("userId="):
-                details["uid"] = line.split("=")[1].strip()
-            elif line.strip().startswith("versionName="):
-                details["version_name"] = line.split("=")[1].strip()
-            elif line.strip().startswith("versionCode="):
-                details["version_code"] = line.split("=", 1)[1].strip()
-            elif line.strip().startswith("timeStamp="):
-                details["timestamp"] = line.split("=")[1].strip()
-            elif line.strip().startswith("firstInstallTime="):
-                details["first_install_time"] = line.split("=")[1].strip()
-            elif line.strip().startswith("lastUpdateTime="):
-                details["last_update_time"] = line.split("=")[1].strip()
-            elif line.strip() == "requested permissions:":
-                in_permissions = True
-                continue
-
-        return details
+        return parse_dumpsys_package_for_details("\n".join(lines))
 
     def _get_files_for_package(self, package_name: str) -> list:
         output = self._adb_command(f"pm path {package_name}")
@@ -235,10 +239,14 @@ class Packages(AndroidExtraction):
         for file_path in output.splitlines():
             file_path = file_path.strip()
 
-            md5 = self._adb_command(f"md5sum {file_path}").split(" ", maxsplit=1)[0]
-            sha1 = self._adb_command(f"sha1sum {file_path}").split(" ", maxsplit=1)[0]
-            sha256 = self._adb_command(f"sha256sum {file_path}").split(" ", maxsplit=1)[0]
-            sha512 = self._adb_command(f"sha512sum {file_path}").split(" ", maxsplit=1)[0]
+            md5 = self._adb_command(
+                f"md5sum {file_path}").split(" ", maxsplit=1)[0]
+            sha1 = self._adb_command(
+                f"sha1sum {file_path}").split(" ", maxsplit=1)[0]
+            sha256 = self._adb_command(
+                f"sha256sum {file_path}").split(" ", maxsplit=1)[0]
+            sha512 = self._adb_command(
+                f"sha512sum {file_path}").split(" ", maxsplit=1)[0]
 
             package_files.append({
                 "path": file_path,
@@ -282,7 +290,8 @@ class Packages(AndroidExtraction):
                 "files": package_files,
             }
 
-            dumpsys_package = self._adb_command(f"dumpsys package {package_name}")
+            dumpsys_package = self._adb_command(
+                f"dumpsys package {package_name}")
             package_details = self.parse_package_for_details(dumpsys_package)
             new_package.update(package_details)
 
@@ -326,9 +335,9 @@ class Packages(AndroidExtraction):
                 continue
 
             packages_to_lookup.append(result)
-            self.log.info("Found non-system package with name \"%s\" installed "
-                          "by \"%s\" on %s", result["package_name"],
-                          result["installer"], result["timestamp"])
+            self.log.info("Found non-system package with name \"%s\" installed by \"%s\" on %s",
+                          result["package_name"], result["installer"],
+                          result["timestamp"])
 
         if not self.fast_mode:
             self.check_virustotal(packages_to_lookup)

mvt/android/modules/adb/processes.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from .base import AndroidExtraction
 
@@ -11,10 +12,15 @@ from .base import AndroidExtraction
 class Processes(AndroidExtraction):
     """This module extracts details on running processes."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -24,7 +30,21 @@ class Processes(AndroidExtraction):
             return
 
         for result in self.results:
-            ioc = self.indicators.check_app_id(result.get("name", ""))
+            proc_name = result.get("proc_name", "")
+            if not proc_name:
+                continue
+
+            # Skipping this process because of false positives.
+            if result["proc_name"] == "gatekeeperd":
+                continue
+
+            ioc = self.indicators.check_app_id(proc_name)
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+            ioc = self.indicators.check_process(proc_name)
             if ioc:
                 result["matched_indicator"] = ioc
                 self.detected.append(result)
@@ -32,7 +52,7 @@ class Processes(AndroidExtraction):
     def run(self) -> None:
         self._adb_connect()
 
-        output = self._adb_command("ps -e")
+        output = self._adb_command("ps -A")
 
         for line in output.splitlines()[1:]:
             line = line.strip()

mvt/android/modules/adb/root_binaries.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from .base import AndroidExtraction
 
@@ -11,10 +12,15 @@ from .base import AndroidExtraction
 class RootBinaries(AndroidExtraction):
     """This module extracts the list of installed packages."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/selinux_status.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from .base import AndroidExtraction
 
@@ -13,10 +14,15 @@ class SELinuxStatus(AndroidExtraction):
 
     slug = "selinux_status"
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/settings.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from .base import AndroidExtraction
 
@@ -59,10 +60,15 @@ ANDROID_DANGEROUS_SETTINGS = [
 class Settings(AndroidExtraction):
     """This module extracts Android system settings."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)

mvt/android/modules/adb/sms.py

@@ -1,12 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
 import sqlite3
-from typing import Union
+from typing import Optional, Union
 
 from mvt.android.parsers.backup import (AndroidBackupParsingError,
                                         parse_tar_for_sms)
@@ -43,12 +43,17 @@ FROM sms;
 
 
 class SMS(AndroidExtraction):
-    """This module extracts all SMS messages containing links."""
+    """This module extracts all SMS messages."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -72,8 +77,10 @@ class SMS(AndroidExtraction):
             if "body" not in message:
                 continue
 
-            # TODO: check links exported from the body previously.
-            message_links = check_for_links(message["body"])
+            message_links = message.get("links", [])
+            if message_links == []:
+                message_links = check_for_links(message["body"])
+
             if self.indicators.check_domains(message_links):
                 self.detected.append(message)
 
@@ -101,15 +108,16 @@ class SMS(AndroidExtraction):
             message["direction"] = ("received" if message["incoming"] == 1 else "sent")
             message["isodate"] = convert_unix_to_iso(message["timestamp"])
 
-            # If we find links in the messages or if they are empty we add
-            # them to the list of results.
-            if check_for_links(message["body"]) or message["body"].strip() == "":
-                self.results.append(message)
+            # Extract links in the message body
+            links = check_for_links(message["body"])
+            message["links"] = links
+
+            self.results.append(message)
 
         cur.close()
         conn.close()
 
-        self.log.info("Extracted a total of %d SMS messages containing links",
+        self.log.info("Extracted a total of %d SMS messages",
                       len(self.results))
 
     def _extract_sms_adb(self) -> None:
@@ -132,7 +140,7 @@ class SMS(AndroidExtraction):
                           "Android Backup Extractor")
             return
 
-        self.log.info("Extracted a total of %d SMS messages containing links",
+        self.log.info("Extracted a total of %d SMS messages",
                       len(self.results))
 
     def run(self) -> None:
@@ -153,7 +161,7 @@ class SMS(AndroidExtraction):
         except InsufficientPrivileges:
             pass
 
-        self.log.warn("No SMS database found. Trying extraction of SMS data "
+        self.log.info("No SMS database found. Trying extraction of SMS data "
                       "using Android backup feature.")
         self._extract_sms_adb()
 

mvt/android/modules/adb/whatsapp.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -7,7 +7,7 @@ import base64
 import logging
 import os
 import sqlite3
-from typing import Union
+from typing import Optional, Union
 
 from mvt.common.utils import check_for_links, convert_unix_to_iso
 
@@ -19,10 +19,15 @@ WHATSAPP_PATH = "data/data/com.whatsapp/databases/msgstore.db"
 class Whatsapp(AndroidExtraction):
     """This module extracts all WhatsApp messages containing links."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -75,17 +80,19 @@ class Whatsapp(AndroidExtraction):
 
             # If we find links in the messages or if they are empty we add them
             # to the list.
-            if check_for_links(message["data"]) or message["data"].strip() == "":
+            if (check_for_links(message["data"])
+                    or message["data"].strip() == ""):
                 if message.get("thumb_image"):
-                    message["thumb_image"] = base64.b64encode(message["thumb_image"])
+                    message["thumb_image"] = base64.b64encode(
+                        message["thumb_image"])
 
                 messages.append(message)
 
         cur.close()
         conn.close()
 
-        self.log.info("Extracted a total of %d WhatsApp messages "
-                      "containing links", len(messages))
+        self.log.info("Extracted a total of %d WhatsApp messages containing links",
+                      len(messages))
         self.results = messages
 
     def run(self) -> None:

mvt/android/modules/androidqf/__init__.py

@@ -0,0 +1,18 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from .dumpsys_accessibility import DumpsysAccessibility
+from .dumpsys_activities import DumpsysActivities
+from .dumpsys_appops import DumpsysAppops
+from .dumpsys_packages import DumpsysPackages
+from .dumpsys_receivers import DumpsysReceivers
+from .getprop import Getprop
+from .processes import Processes
+from .settings import Settings
+from .sms import SMS
+
+ANDROIDQF_MODULES = [DumpsysActivities, DumpsysReceivers, DumpsysAccessibility,
+                     DumpsysAppops, Processes, Getprop, Settings, SMS,
+                     DumpsysPackages]

mvt/android/modules/androidqf/base.py

@@ -0,0 +1,38 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import fnmatch
+import logging
+import os
+from typing import Any, Dict, List, Optional, Union
+
+from mvt.common.module import MVTModule
+
+
+class AndroidQFModule(MVTModule):
+    """This class provides a base for all Android Data analysis modules."""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Union[List[Dict[str, Any]], Dict[str, Any], None] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+        self._path = target_path
+        self._files = []
+
+        for root, dirs, files in os.walk(target_path):
+            for name in files:
+                self._files.append(os.path.join(root, name))
+
+    def _get_files_by_pattern(self, pattern):
+        return fnmatch.filter(self._files, pattern)

mvt/android/modules/androidqf/dumpsys_accessibility.py

@@ -0,0 +1,68 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.android.parsers import parse_dumpsys_accessibility
+
+from .base import AndroidQFModule
+
+
+class DumpsysAccessibility(AndroidQFModule):
+    """This module analyse dumpsys accessbility"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_app_id(result["package_name"])
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+
+    def run(self) -> None:
+        dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt")
+        if not dumpsys_file:
+            return
+
+        lines = []
+        in_accessibility = False
+        with open(dumpsys_file[0]) as handle:
+            for line in handle:
+                if line.strip().startswith("DUMP OF SERVICE accessibility:"):
+                    in_accessibility = True
+                    continue
+
+                if not in_accessibility:
+                    continue
+
+                if line.strip().startswith("-------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
+                    break
+
+                lines.append(line.rstrip())
+
+        self.results = parse_dumpsys_accessibility("\n".join(lines))
+
+        for result in self.results:
+            self.log.info("Found installed accessibility service \"%s\"",
+                          result.get("service"))
+
+        self.log.info("Identified a total of %d accessibility services",
+                      len(self.results))

mvt/android/modules/androidqf/dumpsys_activities.py

@@ -0,0 +1,66 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.android.parsers import parse_dumpsys_activity_resolver_table
+
+from .base import AndroidQFModule
+
+
+class DumpsysActivities(AndroidQFModule):
+    """This module extracts details on receivers for risky activities."""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+        self.results = results if results else {}
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for intent, activities in self.results.items():
+            for activity in activities:
+                ioc = self.indicators.check_app_id(activity["package_name"])
+                if ioc:
+                    activity["matched_indicator"] = ioc
+                    self.detected.append({intent: activity})
+
+    def run(self) -> None:
+        dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt")
+        if not dumpsys_file:
+            return
+
+        lines = []
+        in_package = False
+        with open(dumpsys_file[0]) as handle:
+            for line in handle:
+                if line.strip() == "DUMP OF SERVICE package:":
+                    in_package = True
+                    continue
+
+                if not in_package:
+                    continue
+
+                if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
+                    break
+
+                lines.append(line.rstrip())
+
+        self.results = parse_dumpsys_activity_resolver_table("\n".join(lines))
+
+        self.log.info("Extracted activities for %d intents", len(self.results))

mvt/android/modules/androidqf/dumpsys_appops.py

@@ -0,0 +1,83 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional, Union
+
+from mvt.android.parsers import parse_dumpsys_appops
+
+from .base import AndroidQFModule
+
+
+class DumpsysAppops(AndroidQFModule):
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def serialize(self, record: dict) -> Union[dict, list]:
+        records = []
+        for perm in record["permissions"]:
+            if "entries" not in perm:
+                continue
+
+            for entry in perm["entries"]:
+                if "timestamp" in entry:
+                    records.append({
+                        "timestamp": entry["timestamp"],
+                        "module": self.__class__.__name__,
+                        "event": entry["access"],
+                        "data": f"{record['package_name']} access to "
+                                f"{perm['name']} : {entry['access']}",
+                    })
+
+        return records
+
+    def check_indicators(self) -> None:
+        for result in self.results:
+            if self.indicators:
+                ioc = self.indicators.check_app_id(result.get("package_name"))
+                if ioc:
+                    result["matched_indicator"] = ioc
+                    self.detected.append(result)
+                    continue
+
+            for perm in result["permissions"]:
+                if (perm["name"] == "REQUEST_INSTALL_PACKAGES"
+                        and perm["access"] == "allow"):
+                    self.log.info("Package %s with REQUEST_INSTALL_PACKAGES permission",
+                                  result["package_name"])
+
+    def run(self) -> None:
+        dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt")
+        if not dumpsys_file:
+            return
+
+        lines = []
+        in_package = False
+        with open(dumpsys_file[0]) as handle:
+            for line in handle:
+                if line.startswith("DUMP OF SERVICE appops:"):
+                    in_package = True
+                    continue
+
+                if in_package:
+                    if line.startswith("-------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
+                        break
+
+                    lines.append(line.rstrip())
+
+        self.results = parse_dumpsys_appops("\n".join(lines))
+        self.log.info("Identified %d applications in AppOps Manager",
+                      len(self.results))

mvt/android/modules/androidqf/dumpsys_packages.py

@@ -0,0 +1,106 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from mvt.android.modules.adb.packages import (DANGEROUS_PERMISSIONS,
+                                              DANGEROUS_PERMISSIONS_THRESHOLD,
+                                              ROOT_PACKAGES)
+from mvt.android.parsers.dumpsys import parse_dumpsys_packages
+
+from .base import AndroidQFModule
+
+
+class DumpsysPackages(AndroidQFModule):
+    """This module analyse dumpsys packages"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[List[Dict[str, Any]]] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def serialize(self, record: dict) -> Union[dict, list]:
+        entries = []
+        for entry in ["timestamp", "first_install_time", "last_update_time"]:
+            if entry in record:
+                entries.append({
+                    "timestamp": record[entry],
+                    "module": self.__class__.__name__,
+                    "event": entry,
+                    "data": f"Package {record['package_name']} "
+                            f"({record['uid']})",
+                })
+
+        return entries
+
+    def check_indicators(self) -> None:
+        for result in self.results:
+            if result["package_name"] in ROOT_PACKAGES:
+                self.log.warning("Found an installed package related to "
+                                 "rooting/jailbreaking: \"%s\"",
+                                 result["package_name"])
+                self.detected.append(result)
+                continue
+
+            if not self.indicators:
+                continue
+
+            ioc = self.indicators.check_app_id(result.get("package_name", ""))
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+
+    def run(self) -> None:
+        dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt")
+        if len(dumpsys_file) != 1:
+            self.log.info("Dumpsys file not found")
+            return
+
+        with open(dumpsys_file[0]) as handle:
+            data = handle.read().split("\n")
+
+        package = []
+        in_service = False
+        in_package_list = False
+        for line in data:
+            if line.strip().startswith("DUMP OF SERVICE package:"):
+                in_service = True
+                continue
+
+            if in_service and line.startswith("Packages:"):
+                in_package_list = True
+                continue
+
+            if not in_service or not in_package_list:
+                continue
+
+            if line.strip() == "":
+                break
+
+            package.append(line)
+
+        self.results = parse_dumpsys_packages("\n".join(package))
+
+        for result in self.results:
+            dangerous_permissions_count = 0
+            for perm in result["permissions"]:
+                if perm["name"] in DANGEROUS_PERMISSIONS:
+                    dangerous_permissions_count += 1
+
+            if dangerous_permissions_count >= DANGEROUS_PERMISSIONS_THRESHOLD:
+                self.log.info("Found package \"%s\" requested %d potentially dangerous permissions",
+                              result["package_name"],
+                              dangerous_permissions_count)
+
+        self.log.info("Extracted details on %d packages", len(self.results))

mvt/android/modules/androidqf/dumpsys_receivers.py

@@ -0,0 +1,86 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from mvt.android.modules.adb.dumpsys_receivers import (
+    INTENT_DATA_SMS_RECEIVED, INTENT_NEW_OUTGOING_CALL,
+    INTENT_NEW_OUTGOING_SMS, INTENT_PHONE_STATE, INTENT_SMS_RECEIVED)
+from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
+
+from .base import AndroidQFModule
+
+
+class DumpsysReceivers(AndroidQFModule):
+    """This module analyse dumpsys receivers"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Union[List[Any], Dict[str, Any], None] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+        self.results = results if results else {}
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for intent, receivers in self.results.items():
+            for receiver in receivers:
+                if intent == INTENT_NEW_OUTGOING_SMS:
+                    self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_SMS_RECEIVED:
+                    self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_DATA_SMS_RECEIVED:
+                    self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_PHONE_STATE:
+                    self.log.info("Found a receiver monitoring "
+                                  "telephony state/incoming calls: \"%s\"",
+                                  receiver["receiver"])
+                elif intent == INTENT_NEW_OUTGOING_CALL:
+                    self.log.info("Found a receiver monitoring outgoing calls: \"%s\"",
+                                  receiver["receiver"])
+
+                ioc = self.indicators.check_app_id(receiver["package_name"])
+                if ioc:
+                    receiver["matched_indicator"] = ioc
+                    self.detected.append({intent: receiver})
+
+    def run(self) -> None:
+        dumpsys_file = self._get_files_by_pattern("*/dumpsys.txt")
+        if not dumpsys_file:
+            return
+
+        in_receivers = False
+        lines = []
+        with open(dumpsys_file[0]) as handle:
+            for line in handle:
+                if line.strip() == "DUMP OF SERVICE package:":
+                    in_receivers = True
+                    continue
+
+                if not in_receivers:
+                    continue
+
+                if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
+                    break
+
+                lines.append(line.rstrip())
+
+        self.results = parse_dumpsys_receiver_resolver_table("\n".join(lines))
+
+        self.log.info("Extracted receivers for %d intents", len(self.results))

mvt/android/modules/androidqf/getprop.py

@@ -0,0 +1,76 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from datetime import datetime, timedelta
+from typing import Optional
+
+from mvt.android.parsers.getprop import parse_getprop
+
+from .base import AndroidQFModule
+
+INTERESTING_PROPERTIES = [
+    "gsm.sim.operator.alpha",
+    "gsm.sim.operator.iso-country",
+    "persist.sys.timezone",
+    "ro.boot.serialno",
+    "ro.build.version.sdk",
+    "ro.build.version.security_patch",
+    "ro.product.cpu.abi",
+    "ro.product.locale",
+    "ro.product.vendor.manufacturer",
+    "ro.product.vendor.model",
+    "ro.product.vendor.name"
+]
+
+
+class Getprop(AndroidQFModule):
+    """This module extracts data from get properties."""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+        self.results = []
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            ioc = self.indicators.check_android_property_name(result.get("name", ""))
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+
+    def run(self) -> None:
+        getprop_files = self._get_files_by_pattern("*/getprop.txt")
+        if not getprop_files:
+            self.log.info("getprop.txt file not found")
+            return
+
+        with open(getprop_files[0]) as f:
+            data = f.read()
+
+        self.results = parse_getprop(data)
+        for entry in self.results:
+            if entry["name"] in INTERESTING_PROPERTIES:
+                self.log.info("%s: %s", entry["name"], entry["value"])
+            if entry["name"] == "ro.build.version.security_patch":
+                last_patch = datetime.strptime(entry["value"], "%Y-%m-%d")
+                if (datetime.now() - last_patch) > timedelta(days=6*31):
+                    self.log.warning("This phone has not received security "
+                                     "updates for more than six months "
+                                     "(last update: %s)", entry["value"])
+
+        self.log.info("Extracted a total of %d properties", len(self.results))

mvt/android/modules/androidqf/processes.py

@@ -0,0 +1,92 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from .base import AndroidQFModule
+
+
+class Processes(AndroidQFModule):
+    """This module analyse running processes"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            proc_name = result.get("proc_name", "")
+            if not proc_name:
+                continue
+
+            # Skipping this process because of false positives.
+            if result["proc_name"] == "gatekeeperd":
+                continue
+
+            ioc = self.indicators.check_app_id(proc_name)
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+                continue
+
+            ioc = self.indicators.check_process(proc_name)
+            if ioc:
+                result["matched_indicator"] = ioc
+                self.detected.append(result)
+
+    def _parse_ps(self, data):
+        for line in data.split("\n")[1:]:
+            proc = line.split()
+
+            # Sometimes WCHAN is empty.
+            if len(proc) == 8:
+                proc = proc[:5] + [''] + proc[5:]
+
+            # Sometimes there is the security label.
+            if proc[0].startswith("u:r"):
+                label = proc[0]
+                proc = proc[1:]
+            else:
+                label = ""
+
+            # Sometimes there is no WCHAN.
+            if len(proc) < 9:
+                proc = proc[:5] + [""] + proc[5:]
+
+            self.results.append({
+                "user": proc[0],
+                "pid": int(proc[1]),
+                "ppid": int(proc[2]),
+                "virtual_memory_size": int(proc[3]),
+                "resident_set_size": int(proc[4]),
+                "wchan": proc[5],
+                "aprocress": proc[6],
+                "stat": proc[7],
+                "proc_name": proc[8].strip("[]"),
+                "label": label,
+            })
+
+    def run(self) -> None:
+        ps_files = self._get_files_by_pattern("*/ps.txt")
+        if not ps_files:
+            return
+
+        with open(ps_files[0]) as handle:
+            self._parse_ps(handle.read())
+
+        self.log.info("Identified %d running processes", len(self.results))

mvt/android/modules/androidqf/settings.py

@@ -0,0 +1,58 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import logging
+from typing import Optional
+
+from mvt.android.modules.adb.settings import ANDROID_DANGEROUS_SETTINGS
+
+from .base import AndroidQFModule
+
+
+class Settings(AndroidQFModule):
+    """This module analyse setting files"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+        self.results = {}
+
+    def run(self) -> None:
+        for setting_file in self._get_files_by_pattern("*/settings_*.txt"):
+            namespace = setting_file[setting_file.rfind("_")+1:-4]
+
+            self.results[namespace] = {}
+
+            with open(setting_file) as handle:
+                for line in handle:
+                    line = line.strip()
+                    try:
+                        key, value = line.split("=", 1)
+                    except ValueError:
+                        continue
+
+                    try:
+                        self.results[namespace][key] = value
+                    except IndexError:
+                        continue
+
+                    for danger in ANDROID_DANGEROUS_SETTINGS:
+                        if (danger["key"] == key
+                                and danger["safe_value"] != value):
+                            self.log.warning("Found suspicious setting \"%s = %s\" (%s)",
+                                             key, value, danger["description"])
+                            break
+
+        self.log.info("Identified %d settings",
+                      sum([len(val) for val in self.results.values()]))

mvt/android/modules/androidqf/sms.py

@@ -0,0 +1,85 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 Claudio Guarnieri.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1
+
+import getpass
+import logging
+from typing import Optional
+
+from mvt.android.parsers.backup import (AndroidBackupParsingError,
+                                        InvalidBackupPassword, parse_ab_header,
+                                        parse_backup_file, parse_tar_for_sms)
+
+from .base import AndroidQFModule
+
+
+class SMS(AndroidQFModule):
+    """This module analyse SMS file in backup"""
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
+        super().__init__(file_path=file_path, target_path=target_path,
+                         results_path=results_path, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def check_indicators(self) -> None:
+        if not self.indicators:
+            return
+
+        for message in self.results:
+            if "body" not in message:
+                continue
+
+            if self.indicators.check_domains(message["links"]):
+                self.detected.append(message)
+
+    def parse_backup(self, data):
+        header = parse_ab_header(data)
+        if not header["backup"]:
+            self.log.critical("Invalid backup format, backup.ab was not analysed")
+            return
+
+        password = None
+        if header["encryption"] != "none":
+            password = getpass.getpass(prompt="Backup Password: ", stream=None)
+        try:
+            tardata = parse_backup_file(data, password=password)
+        except InvalidBackupPassword:
+            self.log.critical("Invalid backup password")
+            return
+        except AndroidBackupParsingError:
+            self.log.critical("Impossible to parse this backup file, please use"
+                              " Android Backup Extractor instead")
+            return
+
+        if not tardata:
+            return
+
+        try:
+            self.results = parse_tar_for_sms(tardata)
+        except AndroidBackupParsingError:
+            self.log.info("Impossible to read SMS from the Android Backup, "
+                          "please extract the SMS and try extracting it with "
+                          "Android Backup Extractor")
+            return
+
+    def run(self) -> None:
+        files = self._get_files_by_pattern("*/backup.ab")
+        if not files:
+            self.log.info("No backup data found")
+            return
+
+        with open(files[0], "rb") as handle:
+            data = handle.read()
+
+        self.parse_backup(data)
+        self.log.info("Identified %d SMS in backup data",
+                      len(self.results))

mvt/android/modules/backup/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/modules/backup/base.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -7,6 +7,7 @@ import fnmatch
 import logging
 import os
 from tarfile import TarFile
+from typing import List, Optional
 
 from mvt.common.module import MVTModule
 
@@ -14,27 +15,31 @@ from mvt.common.module import MVTModule
 class BackupExtraction(MVTModule):
     """This class provides a base for all backup extractios modules"""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
-
         self.ab = None
         self.backup_path = None
         self.tar = None
         self.files = []
 
-    def from_folder(self, backup_path: str, files: list) -> None:
+    def from_folder(self, backup_path: Optional[str], files: List[str]) -> None:
         """
         Get all the files and list them
         """
         self.backup_path = backup_path
         self.files = files
 
-    def from_ab(self, file_path: str, tar: TarFile, files: list) -> None:
+    def from_ab(self, file_path: Optional[str], tar: Optional[TarFile], files: List[str]) -> None:
         """
         Extract the files
         """

mvt/android/modules/backup/sms.py

@@ -1,19 +1,27 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.modules.backup.base import BackupExtraction
 from mvt.android.parsers.backup import parse_sms_file
+from mvt.common.utils import check_for_links
 
 
 class SMS(BackupExtraction):
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -27,7 +35,11 @@ class SMS(BackupExtraction):
             if "body" not in message:
                 continue
 
-            if self.indicators.check_domains(message["links"]):
+            message_links = message.get("links", [])
+            if message_links == []:
+                message_links = check_for_links(message.get("text", ""))
+
+            if self.indicators.check_domains(message_links):
                 self.detected.append(message)
 
     def run(self) -> None:
@@ -43,5 +55,5 @@ class SMS(BackupExtraction):
             data = self._get_file_content(file)
             self.results.extend(parse_sms_file(data))
 
-        self.log.info("Extracted a total of %d SMS & MMS messages "
-                      "containing links", len(self.results))
+        self.log.info("Extracted a total of %d SMS & MMS messages",
+                      len(self.results))

mvt/android/modules/bugreport/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/modules/bugreport/accessibility.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_accessibility
 
@@ -13,10 +14,15 @@ from .base import BugReportModule
 class Accessibility(BugReportModule):
     """This module extracts stats on accessibility."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -35,8 +41,8 @@ class Accessibility(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         lines = []
@@ -49,7 +55,7 @@ class Accessibility(BugReportModule):
             if not in_accessibility:
                 continue
 
-            if line.strip().startswith("------------------------------------------------------------------------------"):
+            if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
                 break
 
             lines.append(line)

mvt/android/modules/bugreport/activities.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_activity_resolver_table
 
@@ -13,10 +14,15 @@ from .base import BugReportModule
 class Activities(BugReportModule):
     """This module extracts details on receivers for risky activities."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -38,8 +44,8 @@ class Activities(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         lines = []
@@ -52,7 +58,7 @@ class Activities(BugReportModule):
             if not in_package:
                 continue
 
-            if line.strip().startswith("------------------------------------------------------------------------------"):
+            if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
                 break
 
             lines.append(line)

mvt/android/modules/bugreport/appops.py

@@ -1,10 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-from typing import Union
+from typing import Optional, Union
 
 from mvt.android.parsers import parse_dumpsys_appops
 
@@ -14,10 +14,15 @@ from .base import BugReportModule
 class Appops(BugReportModule):
     """This module extracts information on package from App-Ops Manager."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -58,8 +63,8 @@ class Appops(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         lines = []
@@ -72,7 +77,7 @@ class Appops(BugReportModule):
             if not in_appops:
                 continue
 
-            if line.strip().startswith("------------------------------------------------------------------------------"):
+            if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
                 break
 
             lines.append(line)

mvt/android/modules/bugreport/base.py

@@ -1,11 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
 import fnmatch
 import logging
 import os
+from typing import List, Optional
 from zipfile import ZipFile
 
 from mvt.common.module import MVTModule
@@ -14,24 +15,29 @@ from mvt.common.module import MVTModule
 class BugReportModule(MVTModule):
     """This class provides a base for all Android Bug Report modules."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
 
-        self.zip_archive = None
-        self.extract_path = None
-        self.extract_files = []
-        self.zip_files = []
+        self.zip_archive: Optional[ZipFile] = None
+        self.extract_path: Optional[str] = None
+        self.extract_files: List[str] = []
+        self.zip_files: List[str] = []
 
-    def from_folder(self, extract_path: str, extract_files: str) -> None:
+    def from_folder(self, extract_path: Optional[str], extract_files: List[str]) -> None:
         self.extract_path = extract_path
         self.extract_files = extract_files
 
-    def from_zip(self, zip_archive: ZipFile, zip_files: list) -> None:
+    def from_zip(self, zip_archive: Optional[ZipFile], zip_files: List[str]) -> None:
         self.zip_archive = zip_archive
         self.zip_files = zip_files
 
@@ -51,6 +57,8 @@ class BugReportModule(MVTModule):
             if matches:
                 return matches
 
+        return []
+
     def _get_file_content(self, file_path: str) -> bytes:
         if self.zip_archive:
             handle = self.zip_archive.open(file_path)

mvt/android/modules/bugreport/battery_daily.py

@@ -1,10 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-from typing import Union
+from typing import Optional, Union
 
 from mvt.android.parsers import parse_dumpsys_battery_daily
 
@@ -14,10 +14,15 @@ from .base import BugReportModule
 class BatteryDaily(BugReportModule):
     """This module extracts records from battery daily updates."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -45,8 +50,8 @@ class BatteryDaily(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         lines = []

mvt/android/modules/bugreport/battery_history.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_battery_history
 
@@ -13,10 +14,15 @@ from .base import BugReportModule
 class BatteryHistory(BugReportModule):
     """This module extracts records from battery daily updates."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -35,8 +41,8 @@ class BatteryHistory(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide "
-                           "a valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         lines = []

mvt/android/modules/bugreport/dbinfo.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_dbinfo
 
@@ -15,10 +16,15 @@ class DBInfo(BugReportModule):
 
     slug = "dbinfo"
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -39,8 +45,8 @@ class DBInfo(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         in_dbinfo = False
@@ -53,7 +59,7 @@ class DBInfo(BugReportModule):
             if not in_dbinfo:
                 continue
 
-            if line.strip().startswith("------------------------------------------------------------------------------"):
+            if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
                 break
 
             lines.append(line)

mvt/android/modules/bugreport/getprop.py

@@ -1,10 +1,11 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 from datetime import datetime, timedelta
+from typing import Optional
 
 from mvt.android.parsers import parse_getprop
 
@@ -14,10 +15,15 @@ from .base import BugReportModule
 class Getprop(BugReportModule):
     """This module extracts device properties from getprop command."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -27,8 +33,8 @@ class Getprop(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         lines = []

mvt/android/modules/bugreport/packages.py

@@ -1,15 +1,15 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
-import re
-from typing import Union
+from typing import Optional, Union
 
 from mvt.android.modules.adb.packages import (DANGEROUS_PERMISSIONS,
                                               DANGEROUS_PERMISSIONS_THRESHOLD,
                                               ROOT_PACKAGES)
+from mvt.android.parsers.dumpsys import parse_dumpsys_packages
 
 from .base import BugReportModule
 
@@ -17,10 +17,15 @@ from .base import BugReportModule
 class Packages(BugReportModule):
     """This module extracts details on receivers for risky activities."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -71,94 +76,11 @@ class Packages(BugReportModule):
                 self.detected.append(result)
                 continue
 
-    @staticmethod
-    def parse_package_for_details(output: str) -> dict:
-        details = {
-            "uid": "",
-            "version_name": "",
-            "version_code": "",
-            "timestamp": "",
-            "first_install_time": "",
-            "last_update_time": "",
-            "requested_permissions": [],
-        }
-
-        in_install_permissions = False
-        in_runtime_permissions = False
-        for line in output.splitlines():
-            if in_install_permissions:
-                if line.startswith(" " * 4) and not line.startswith(" " * 6):
-                    in_install_permissions = False
-                    continue
-
-                permission = line.strip().split(":")[0]
-                if permission not in details["requested_permissions"]:
-                    details["requested_permissions"].append(permission)
-
-            if in_runtime_permissions:
-                if not line.startswith(" " * 8):
-                    in_runtime_permissions = False
-                    continue
-
-                permission = line.strip().split(":")[0]
-                if permission not in details["requested_permissions"]:
-                    details["requested_permissions"].append(permission)
-
-            if line.strip().startswith("userId="):
-                details["uid"] = line.split("=")[1].strip()
-            elif line.strip().startswith("versionName="):
-                details["version_name"] = line.split("=")[1].strip()
-            elif line.strip().startswith("versionCode="):
-                details["version_code"] = line.split("=", 1)[1].strip()
-            elif line.strip().startswith("timeStamp="):
-                details["timestamp"] = line.split("=")[1].strip()
-            elif line.strip().startswith("firstInstallTime="):
-                details["first_install_time"] = line.split("=")[1].strip()
-            elif line.strip().startswith("lastUpdateTime="):
-                details["last_update_time"] = line.split("=")[1].strip()
-            elif line.strip() == "install permissions:":
-                in_install_permissions = True
-            elif line.strip() == "runtime permissions:":
-                in_runtime_permissions = True
-
-        return details
-
-    def parse_packages_list(self, output: str) -> list:
-        pkg_rxp = re.compile(r"  Package \[(.+?)\].*")
-
-        results = []
-        package_name = None
-        package = {}
-        lines = []
-        for line in output.splitlines():
-            if line.startswith("  Package ["):
-                if len(lines) > 0:
-                    details = self.parse_package_for_details("\n".join(lines))
-                    package.update(details)
-                    results.append(package)
-                    lines = []
-                    package = {}
-
-                matches = pkg_rxp.findall(line)
-                if not matches:
-                    continue
-
-                package_name = matches[0]
-                package["package_name"] = package_name
-                continue
-
-            if not package_name:
-                continue
-
-            lines.append(line)
-
-        return results
-
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         in_package = False
@@ -184,17 +106,17 @@ class Packages(BugReportModule):
 
             lines.append(line)
 
-        self.results = self.parse_packages_list("\n".join(lines))
+        self.results = parse_dumpsys_packages("\n".join(lines))
 
         for result in self.results:
             dangerous_permissions_count = 0
-            for perm in result["requested_permissions"]:
-                if perm in DANGEROUS_PERMISSIONS:
+            for perm in result["permissions"]:
+                if perm["name"] in DANGEROUS_PERMISSIONS:
                     dangerous_permissions_count += 1
 
             if dangerous_permissions_count >= DANGEROUS_PERMISSIONS_THRESHOLD:
-                self.log.info("Found package \"%s\" requested %d potentially "
-                              "dangerous permissions", result["package_name"],
+                self.log.info("Found package \"%s\" requested %d potentially dangerous permissions",
+                              result["package_name"],
                               dangerous_permissions_count)
 
         self.log.info("Extracted details on %d packages", len(self.results))

mvt/android/modules/bugreport/receivers.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
 
@@ -19,10 +20,15 @@ INTENT_NEW_OUTGOING_CALL = "android.intent.action.NEW_OUTGOING_CALL"
 class Receivers(BugReportModule):
     """This module extracts details on receivers for risky activities."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -36,24 +42,20 @@ class Receivers(BugReportModule):
         for intent, receivers in self.results.items():
             for receiver in receivers:
                 if intent == INTENT_NEW_OUTGOING_SMS:
-                    self.log.info("Found a receiver to intercept "
-                                  "outgoing SMS messages: \"%s\"",
+                    self.log.info("Found a receiver to intercept outgoing SMS messages: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_SMS_RECEIVED:
-                    self.log.info("Found a receiver to intercept "
-                                  "incoming SMS messages: \"%s\"",
+                    self.log.info("Found a receiver to intercept incoming SMS messages: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_DATA_SMS_RECEIVED:
-                    self.log.info("Found a receiver to intercept "
-                                  "incoming data SMS message: \"%s\"",
+                    self.log.info("Found a receiver to intercept incoming data SMS message: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_PHONE_STATE:
                     self.log.info("Found a receiver monitoring "
                                   "telephony state/incoming calls: \"%s\"",
                                   receiver["receiver"])
                 elif intent == INTENT_NEW_OUTGOING_CALL:
-                    self.log.info("Found a receiver monitoring "
-                                  "outgoing calls: \"%s\"",
+                    self.log.info("Found a receiver monitoring outgoing calls: \"%s\"",
                                   receiver["receiver"])
 
                 ioc = self.indicators.check_app_id(receiver["package_name"])
@@ -65,8 +67,8 @@ class Receivers(BugReportModule):
     def run(self) -> None:
         content = self._get_dumpstate_file()
         if not content:
-            self.log.error("Unable to find dumpstate file. Did you provide a "
-                           "valid bug report archive?")
+            self.log.error("Unable to find dumpstate file. "
+                           "Did you provide a valid bug report archive?")
             return
 
         in_receivers = False
@@ -79,7 +81,7 @@ class Receivers(BugReportModule):
             if not in_receivers:
                 continue
 
-            if line.strip().startswith("------------------------------------------------------------------------------"):
+            if line.strip().startswith("------------------------------------------------------------------------------"):  # pylint: disable=line-too-long
                 break
 
             lines.append(line)

mvt/android/parsers/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/android/parsers/backup.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -218,10 +218,9 @@ def parse_sms_file(data):
         entry["isodate"] = convert_unix_to_iso(int(entry["date"]) / 1000)
         entry["direction"] = ("sent" if int(entry["date_sent"]) else "received")
 
-        # If we find links in the messages or if they are empty we add them to
-        # the list.
+        # Extract links from the body
         if message_links or entry["body"].strip() == "":
             entry["links"] = message_links
-            res.append(entry)
+        res.append(entry)
 
     return res

mvt/android/parsers/dumpsys.py

@@ -1,15 +1,16 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import re
 from datetime import datetime
+from typing import Any, Dict, List
 
 from mvt.common.utils import convert_datetime_to_iso
 
 
-def parse_dumpsys_accessibility(output: str) -> list:
+def parse_dumpsys_accessibility(output: str) -> List[Dict[str, str]]:
     results = []
 
     in_services = False
@@ -34,7 +35,7 @@ def parse_dumpsys_accessibility(output: str) -> list:
     return results
 
 
-def parse_dumpsys_activity_resolver_table(output: str) -> dict:
+def parse_dumpsys_activity_resolver_table(output: str) -> Dict[str, Any]:
     results = {}
 
     in_activity_resolver_table = False
@@ -138,7 +139,7 @@ def parse_dumpsys_battery_daily(output: str) -> list:
     return results
 
 
-def parse_dumpsys_battery_history(output: str) -> list:
+def parse_dumpsys_battery_history(output: str) -> List[Dict[str, Any]]:
     results = []
 
     for line in output.splitlines():
@@ -169,6 +170,17 @@ def parse_dumpsys_battery_history(output: str) -> list:
                 continue
 
             package_name = service.split("/")[0]
+        elif (line.find("+top=") > 0) or (line.find("-top") > 0):
+            if line.find("+top=") > 0:
+                event = "start_top"
+                top_pos = line.find("+top=")
+            else:
+                event = "end_top"
+                top_pos = line.find("-top=")
+            colon_pos = top_pos+line[top_pos:].find(":")
+            uid = line[top_pos+5:colon_pos]
+            service = ""
+            package_name = line[colon_pos+1:].strip('"')
         else:
             continue
 
@@ -183,11 +195,11 @@ def parse_dumpsys_battery_history(output: str) -> list:
     return results
 
 
-def parse_dumpsys_dbinfo(output: str) -> list:
+def parse_dumpsys_dbinfo(output: str) -> List[Dict[str, Any]]:
     results = []
 
-    rxp = re.compile(r'.*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\].*\[Pid:\((\d+)\)\](\w+).*sql\=\"(.+?)\"')
-    rxp_no_pid = re.compile(r'.*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\][ ]{1}(\w+).*sql\=\"(.+?)\"')
+    rxp = re.compile(r'.*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\].*\[Pid:\((\d+)\)\](\w+).*sql\=\"(.+?)\"')  # pylint: disable=line-too-long
+    rxp_no_pid = re.compile(r'.*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\][ ]{1}(\w+).*sql\=\"(.+?)\"')  # pylint: disable=line-too-long
 
     pool = None
     in_operations = False
@@ -236,7 +248,7 @@ def parse_dumpsys_dbinfo(output: str) -> list:
     return results
 
 
-def parse_dumpsys_receiver_resolver_table(output: str) -> dict:
+def parse_dumpsys_receiver_resolver_table(output: str) -> Dict[str, Any]:
     results = {}
 
     in_receiver_resolver_table = False
@@ -293,7 +305,7 @@ def parse_dumpsys_receiver_resolver_table(output: str) -> dict:
     return results
 
 
-def parse_dumpsys_appops(output: str) -> list:
+def parse_dumpsys_appops(output: str) -> List[Dict[str, Any]]:
     results = []
     perm = {}
     package = {}
@@ -376,3 +388,134 @@ def parse_dumpsys_appops(output: str) -> list:
         results.append(package)
 
     return results
+
+
+def parse_dumpsys_package_for_details(output: str) -> Dict[str, Any]:
+    """
+    Parse one entry of a dumpsys package information
+    """
+    details = {
+        "uid": "",
+        "version_name": "",
+        "version_code": "",
+        "timestamp": "",
+        "first_install_time": "",
+        "last_update_time": "",
+        "permissions": [],
+        "requested_permissions": [],
+    }
+
+    in_install_permissions = False
+    in_runtime_permissions = False
+    in_declared_permissions = False
+    in_requested_permissions = True
+    for line in output.splitlines():
+        if in_install_permissions:
+            if line.startswith(" " * 4) and not line.startswith(" " * 6):
+                in_install_permissions = False
+            else:
+                lineinfo = line.strip().split(":")
+                permission = lineinfo[0]
+                granted = None
+                if "granted=" in lineinfo[1]:
+                    granted = ("granted=true" in lineinfo[1])
+
+                details["permissions"].append({
+                    "name": permission,
+                    "granted": granted,
+                    "type": "install"
+                })
+
+        if in_runtime_permissions:
+            if not line.startswith(" " * 8):
+                in_runtime_permissions = False
+            else:
+                lineinfo = line.strip().split(":")
+                permission = lineinfo[0]
+                granted = None
+                if "granted=" in lineinfo[1]:
+                    granted = ("granted=true" in lineinfo[1])
+
+                details["permissions"].append({
+                    "name": permission,
+                    "granted": granted,
+                    "type": "runtime"
+                })
+
+        if in_declared_permissions:
+            if not line.startswith(" " * 6):
+                in_declared_permissions = False
+            else:
+                permission = line.strip().split(":")[0]
+                details["permissions"].append({
+                    "name": permission,
+                    "type": "declared"
+                })
+        if in_requested_permissions:
+            if not line.startswith(" " * 6):
+                in_requested_permissions = False
+            else:
+                details["requested_permissions"].append(line.strip())
+
+        if line.strip().startswith("userId="):
+            details["uid"] = line.split("=")[1].strip()
+        elif line.strip().startswith("versionName="):
+            details["version_name"] = line.split("=")[1].strip()
+        elif line.strip().startswith("versionCode="):
+            details["version_code"] = line.split("=", 1)[1].strip()
+        elif line.strip().startswith("timeStamp="):
+            details["timestamp"] = line.split("=")[1].strip()
+        elif line.strip().startswith("firstInstallTime="):
+            details["first_install_time"] = line.split("=")[1].strip()
+        elif line.strip().startswith("lastUpdateTime="):
+            details["last_update_time"] = line.split("=")[1].strip()
+        elif line.strip() == "install permissions:":
+            in_install_permissions = True
+        elif line.strip() == "runtime permissions:":
+            in_runtime_permissions = True
+        elif line.strip() == "declared permissions:":
+            in_declared_permissions = True
+        elif line.strip() == "requested permissions:":
+            in_requested_permissions = True
+
+    return details
+
+
+def parse_dumpsys_packages(output: str) -> List[Dict[str, Any]]:
+    """
+    Parse the dumpsys package service data
+    """
+    pkg_rxp = re.compile(r"  Package \[(.+?)\].*")
+
+    results = []
+    package_name = None
+    package = {}
+    lines = []
+    for line in output.splitlines():
+        if line.startswith("  Package ["):
+            if len(lines) > 0:
+                details = parse_dumpsys_package_for_details("\n".join(lines))
+                package.update(details)
+                results.append(package)
+                lines = []
+                package = {}
+
+            matches = pkg_rxp.findall(line)
+            if not matches:
+                continue
+
+            package_name = matches[0]
+            package["package_name"] = package_name
+            continue
+
+        if not package_name:
+            continue
+
+        lines.append(line)
+
+    if len(lines) > 0:
+        details = parse_dumpsys_package_for_details("\n".join(lines))
+        package.update(details)
+        results.append(package)
+
+    return results

mvt/android/parsers/getprop.py

@@ -1,13 +1,14 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import re
+from typing import Dict, List
 
 
-def parse_getprop(output: str) -> dict:
-    results = {}
+def parse_getprop(output: str) -> List[Dict[str, str]]:
+    results = []
     rxp = re.compile(r"\[(.+?)\]: \[(.+?)\]")
 
     for line in output.splitlines():
@@ -19,8 +20,10 @@ def parse_getprop(output: str) -> dict:
         if not matches or len(matches[0]) != 2:
             continue
 
-        key = matches[0][0]
-        value = matches[0][1]
-        results[key] = value
+        entry = {
+            "name": matches[0][0],
+            "value": matches[0][1]
+        }
+        results.append(entry)
 
     return results

mvt/common/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/common/cmd_check_iocs.py

@@ -1,10 +1,11 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
+from typing import Optional
 
 from mvt.common.command import Command
 
@@ -13,9 +14,15 @@ log = logging.getLogger(__name__)
 
 class CmdCheckIOCS(Command):
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+    ) -> None:
         super().__init__(target_path=target_path, results_path=results_path,
                          ioc_files=ioc_files, module_name=module_name,
                          serial=serial, fast_mode=fast_mode, log=log)
@@ -23,6 +30,7 @@ class CmdCheckIOCS(Command):
         self.name = "check-iocs"
 
     def run(self) -> None:
+        assert self.target_path is not None
         all_modules = []
         for entry in self.modules:
             if entry not in all_modules:
@@ -42,8 +50,8 @@ class CmdCheckIOCS(Command):
                 if iocs_module().get_slug() != name_only:
                     continue
 
-                log.info("Loading results from \"%s\" with module %s", file_name,
-                         iocs_module.__name__)
+                log.info("Loading results from \"%s\" with module %s",
+                         file_name, iocs_module.__name__)
 
                 m = iocs_module.from_json(file_path,
                                           log=logging.getLogger(iocs_module.__module__))

mvt/common/command.py

@@ -1,46 +1,58 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import hashlib
 import json
 import logging
 import os
 import sys
 from datetime import datetime
-from typing import Callable
+from typing import Optional
 
 from mvt.common.indicators import Indicators
-from mvt.common.module import run_module, save_timeline
-from mvt.common.utils import convert_datetime_to_iso
+from mvt.common.module import MVTModule, run_module, save_timeline
+from mvt.common.utils import (convert_datetime_to_iso,
+                              generate_hashes_from_path,
+                              get_sha256_from_file_path)
 from mvt.common.version import MVT_VERSION
 
 
 class Command:
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__)):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        hashes: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+    ) -> None:
         self.name = ""
         self.modules = []
 
         self.target_path = target_path
         self.results_path = results_path
-        self.ioc_files = ioc_files
+        self.ioc_files = ioc_files if ioc_files else []
         self.module_name = module_name
         self.serial = serial
         self.fast_mode = fast_mode
         self.log = log
 
         self.iocs = Indicators(log=log)
-        self.iocs.load_indicators_files(ioc_files)
+        self.iocs.load_indicators_files(self.ioc_files)
 
         # This list will contain all executed modules.
         # We can use this to reference e.g. self.executed[0].results.
         self.executed = []
 
+        self.detected_count = 0
+
+        self.hashes = hashes
+        self.hash_values = []
         self.timeline = []
         self.timeline_detected = []
 
@@ -99,45 +111,29 @@ class Command:
             if ioc_file_path and ioc_file_path not in info["ioc_files"]:
                 info["ioc_files"].append(ioc_file_path)
 
-        # TODO: Revisit if setting this from environment variable is good
-        #       enough.
-        if self.target_path and os.environ.get("MVT_HASH_FILES"):
-            if os.path.isfile(self.target_path):
-                sha256 = hashlib.sha256()
-                with open(self.target_path, "rb") as handle:
-                    sha256.update(handle.read())
+        if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes):
+            self.generate_hashes()
 
-                info["hashes"].append({
-                    "file_path": self.target_path,
-                    "sha256": sha256.hexdigest(),
-                })
-            elif os.path.isdir(self.target_path):
-                for (root, _, files) in os.walk(self.target_path):
-                    for file in files:
-                        file_path = os.path.join(root, file)
-                        sha256 = hashlib.sha256()
-
-                        try:
-                            with open(file_path, "rb") as handle:
-                                sha256.update(handle.read())
-                        except FileNotFoundError:
-                            self.log.error("Failed to hash the file %s: might "
-                                           "be a symlink", file_path)
-                            continue
-                        except PermissionError:
-                            self.log.error("Failed to hash the file %s: "
-                                           "permission denied", file_path)
-                            continue
-
-                        info["hashes"].append({
-                            "file_path": file_path,
-                            "sha256": sha256.hexdigest(),
-                        })
+        info["hashes"] = self.hash_values
 
         info_path = os.path.join(self.results_path, "info.json")
         with open(info_path, "w+", encoding="utf-8") as handle:
             json.dump(info, handle, indent=4)
 
+        if self.target_path and (os.environ.get("MVT_HASH_FILES") or self.hashes):
+            info_hash = get_sha256_from_file_path(info_path)
+            self.log.info("Reference hash of the info.json file: \"%s\"", info_hash)
+
+    def generate_hashes(self) -> None:
+        """
+        Compute hashes for files in the target_path
+        """
+        if not self.target_path:
+            return
+
+        for file in generate_hashes_from_path(self.target_path, self.log):
+            self.hash_values.append(file)
+
     def list_modules(self) -> None:
         self.log.info("Following is the list of available %s modules:",
                       self.name)
@@ -147,7 +143,7 @@ class Command:
     def init(self) -> None:
         raise NotImplementedError
 
-    def module_init(self, module: Callable) -> None:
+    def module_init(self, module: MVTModule) -> None:
         raise NotImplementedError
 
     def finish(self) -> None:
@@ -190,13 +186,15 @@ class Command:
 
             self.executed.append(m)
 
+            self.detected_count += len(m.detected)
+
             self.timeline.extend(m.timeline)
             self.timeline_detected.extend(m.timeline_detected)
 
-        self._store_timeline()
-        self._store_info()
-
         try:
             self.finish()
         except NotImplementedError:
             pass
+
+        self._store_timeline()
+        self._store_info()

mvt/common/help.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -9,6 +9,7 @@ HELP_MSG_IOC = "Path to indicators file (can be invoked multiple time)"
 HELP_MSG_FAST = "Avoid running time/resource consuming features"
 HELP_MSG_LIST_MODULES = "Print list of available modules and exit"
 HELP_MSG_MODULE = "Name of a single module you would like to run instead of all"
+HELP_MSG_HASHES = "Generate hashes of all the files analyzed"
 
 # Android-specific.
 HELP_MSG_SERIAL = "Specify a device serial number or HOST:PORT connection string"

mvt/common/indicators.py

@@ -1,12 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import json
 import logging
 import os
-from typing import Union
+from typing import Any, Dict, Iterator, List, Optional, Union
 
 from appdirs import user_data_dir
 
@@ -15,15 +15,17 @@ from .url import URL
 MVT_DATA_FOLDER = user_data_dir("mvt")
 MVT_INDICATORS_FOLDER = os.path.join(MVT_DATA_FOLDER, "indicators")
 
+logger = logging.getLogger(__name__)
+
 
 class Indicators:
     """This class is used to parse indicators from a STIX2 file and provide
     functions to compare extracted artifacts to the indicators.
     """
 
-    def __init__(self, log=logging.Logger) -> None:
+    def __init__(self, log=logger) -> None:
         self.log = log
-        self.ioc_collections = []
+        self.ioc_collections: List[Dict[str, Any]] = []
         self.total_ioc_count = 0
 
     def _load_downloaded_indicators(self) -> None:
@@ -47,12 +49,17 @@ class Indicators:
             if os.path.isfile(path):
                 self.parse_stix2(path)
             else:
-                self.log.error("Path specified with env MVT_STIX2 is not "
-                               "a valid file: %s", path)
+                self.log.error("Path specified with env MVT_STIX2 is not a valid file: %s",
+                               path)
 
-    def _new_collection(self, cid: str = "", name: str = "",
-                        description: str = "", file_name: str = "",
-                        file_path: str = "") -> dict:
+    def _new_collection(
+        self,
+        cid: Optional[str] = None,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        file_name: Optional[str] = None,
+        file_path: Optional[str] = None
+    ) -> dict:
         return {
             "id": cid,
             "name": name,
@@ -67,6 +74,7 @@ class Indicators:
             "files_sha256": [],
             "app_ids": [],
             "ios_profile_ids": [],
+            "android_property_names": [],
             "count": 0,
         }
 
@@ -116,6 +124,11 @@ class Indicators:
                                 ioc_coll=collection,
                                 ioc_coll_list=collection["ios_profile_ids"])
 
+        elif key == "android-property:name":
+            self._add_indicator(ioc=value,
+                                ioc_coll=collection,
+                                ioc_coll_list=collection["android_property_names"])
+
     def parse_stix2(self, file_path: str) -> None:
         """Extract indicators from a STIX2 file.
 
@@ -130,8 +143,7 @@ class Indicators:
                 data = json.load(handle)
             except json.decoder.JSONDecodeError:
                 self.log.critical("Unable to parse STIX2 indicator file. "
-                                  "The file is corrupted or in the wrong "
-                                  "format!")
+                                  "The file is corrupted or in the wrong format!")
                 return
 
         malware = {}
@@ -186,7 +198,7 @@ class Indicators:
         self.ioc_collections.extend(collections)
 
     def load_indicators_files(self, files: list,
-                              load_default: bool = True) -> None:
+                              load_default: Optional[bool] = True) -> None:
         """
         Load a list of indicators files.
         """
@@ -205,7 +217,7 @@ class Indicators:
         self.log.info("Loaded a total of %d unique indicators",
                       self.total_ioc_count)
 
-    def get_iocs(self, ioc_type: str) -> Union[dict, None]:
+    def get_iocs(self, ioc_type: str) -> Iterator[Dict[str, Any]]:
         for ioc_collection in self.ioc_collections:
             for ioc in ioc_collection.get(ioc_type, []):
                 yield {
@@ -223,10 +235,10 @@ class Indicators:
         :returns: Indicator details if matched, otherwise None
 
         """
-        # TODO: If the IOC domain contains a subdomain, it is not currently
-        #       being matched.
         if not url:
             return None
+        if not isinstance(url, str):
+            return None
 
         try:
             # First we use the provided URL.
@@ -237,15 +249,17 @@ class Indicators:
                 # HTTP HEAD request.
                 unshortened = orig_url.unshorten()
 
-                # self.log.info("Found a shortened URL %s -> %s",
-                #               url, unshortened)
+                self.log.debug("Found a shortened URL %s -> %s",
+                               url, unshortened)
+                if unshortened is None:
+                    return None
 
                 # Now we check for any nested URL shorteners.
                 dest_url = URL(unshortened)
                 if dest_url.check_if_shortened():
-                    # self.log.info("Original URL %s appears to shorten another "
-                    #               "shortened URL %s ... checking!",
-                    #               orig_url.url, dest_url.url)
+                    self.log.debug("Original URL %s appears to shorten another "
+                                   "shortened URL %s ... checking!",
+                                   orig_url.url, dest_url.url)
                     return self.check_domain(dest_url.url)
 
                 final_url = dest_url
@@ -272,9 +286,8 @@ class Indicators:
             if final_url.domain.lower() == ioc["value"]:
                 if orig_url.is_shortened and orig_url.url != final_url.url:
                     self.log.warning("Found a known suspicious domain %s "
-                                     "shortened as %s matching indicators "
-                                     "from \"%s\"", final_url.url, orig_url.url,
-                                     ioc["name"])
+                                     "shortened as %s matching indicators from \"%s\"",
+                                     final_url.url, orig_url.url, ioc["name"])
                 else:
                     self.log.warning("Found a known suspicious domain %s "
                                      "matching indicators from \"%s\"",
@@ -339,8 +352,8 @@ class Indicators:
             if len(proc_name) == 16:
                 if ioc["value"].startswith(proc_name):
                     self.log.warning("Found a truncated known suspicious "
-                                     "process name \"%s\" matching indicators "
-                                     "from \"%s\"", process, ioc["name"])
+                                     "process name \"%s\" matching indicators from \"%s\"",
+                                     process, ioc["name"])
                     return ioc
 
         return None
@@ -377,8 +390,8 @@ class Indicators:
 
         for ioc in self.get_iocs("emails"):
             if email.lower() == ioc["value"].lower():
-                self.log.warning("Found a known suspicious email address \"%s\""
-                                 " matching indicators from \"%s\"",
+                self.log.warning("Found a known suspicious email address \"%s\" "
+                                 "matching indicators from \"%s\"",
                                  email, ioc["name"])
                 return ioc
 
@@ -433,6 +446,29 @@ class Indicators:
 
         return None
 
+    def check_file_path_process(self, file_path: str) -> Optional[Dict[str, Any]]:
+        """Check the provided file path contains a process name from the
+        list of indicators
+
+        :param file_path: File path or file name to check against file
+        indicators
+        :type file_path: str
+        :returns: Indicator details if matched, otherwise None
+
+        """
+        if not file_path:
+            return None
+
+        for ioc in self.get_iocs("processes"):
+            parts = file_path.split("/")
+            if ioc["value"] in parts:
+                self.log.warning("Found known suspicious process name mentioned in file at "
+                                  "path \"%s\" matching indicators from \"%s\"",
+                                  file_path, ioc["name"])
+                return ioc
+
+        return None
+
     def check_profile(self, profile_uuid: str) -> Union[dict, None]:
         """Check the provided configuration profile UUID against the list of
         indicators.
@@ -468,8 +504,8 @@ class Indicators:
 
         for ioc in self.get_iocs("files_sha256"):
             if file_hash.lower() == ioc["value"].lower():
-                self.log.warning("Found a known suspicious file with hash "
-                                 "\"%s\" matching indicators from \"%s\"",
+                self.log.warning("Found a known suspicious file with hash \"%s\" "
+                                 "matching indicators from \"%s\"",
                                  file_hash, ioc["name"])
                 return ioc
 
@@ -495,3 +531,23 @@ class Indicators:
                 return ioc
 
         return None
+
+    def check_android_property_name(self, property_name: str) -> Optional[dict]:
+        """Check the android property name against the list of indicators.
+
+        :param property_name: Name of the Android property
+        :type property_name: str
+        :returns: Indicator details if matched, otherwise None
+
+        """
+        if property_name is None:
+            return None
+
+        for ioc in self.get_iocs("android_property_names"):
+            if property_name.lower() == ioc["value"].lower():
+                self.log.warning("Found a known suspicious Android property \"%s\" "
+                                 "matching indicators from \"%s\"", property_name,
+                                 ioc["name"])
+                return ioc
+
+        return None

mvt/common/logo.py

@@ -1,9 +1,9 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-from rich import print
+from rich import print as rich_print
 
 from .updates import IndicatorsUpdates, MVTUpdates
 from .version import MVT_VERSION
@@ -18,8 +18,8 @@ def check_updates() -> None:
         pass
     else:
         if latest_version:
-            print(f"\t\t[bold]Version {latest_version} is available! "
-                  "Upgrade mvt![/bold]")
+            rich_print(f"\t\t[bold]Version {latest_version} is available! "
+                       "Upgrade mvt with `pip3 install -U mvt`[/bold]")
 
     # Then we check for indicators files updates.
     ioc_updates = IndicatorsUpdates()
@@ -27,8 +27,8 @@ def check_updates() -> None:
     # Before proceeding, we check if we have downloaded an indicators index.
     # If not, there's no point in proceeding with the updates check.
     if ioc_updates.get_latest_update() == 0:
-        print("\t\t[bold]You have not yet downloaded any indicators, check "
-              "the `download-iocs` command![/bold]")
+        rich_print("\t\t[bold]You have not yet downloaded any indicators, check "
+                   "the `download-iocs` command![/bold]")
         return
 
     # We only perform this check at a fixed frequency, in order to not
@@ -36,8 +36,8 @@ def check_updates() -> None:
     # multiple times.
     should_check, hours = ioc_updates.should_check()
     if not should_check:
-        print(f"\t\tIndicators updates checked recently, next automatic check "
-              f"in {int(hours)} hours")
+        rich_print(f"\t\tIndicators updates checked recently, next automatic check "
+                   f"in {int(hours)} hours")
         return
 
     try:
@@ -46,18 +46,18 @@ def check_updates() -> None:
         pass
     else:
         if ioc_to_update:
-            print("\t\t[bold]There are updates to your indicators files! "
-                  "Run the `download-iocs` command to update![/bold]")
+            rich_print("\t\t[bold]There are updates to your indicators files! "
+                       "Run the `download-iocs` command to update![/bold]")
         else:
-            print("\t\tYour indicators files seem to be up to date.")
+            rich_print("\t\tYour indicators files seem to be up to date.")
 
 
 def logo() -> None:
-    print("\n")
-    print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
-    print("\t\thttps://mvt.re")
-    print(f"\t\tVersion: {MVT_VERSION}")
+    rich_print("\n")
+    rich_print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
+    rich_print("\t\thttps://mvt.re")
+    rich_print(f"\t\tVersion: {MVT_VERSION}")
 
     check_updates()
 
-    print("\n")
+    rich_print("\n")

mvt/common/module.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -7,7 +7,7 @@ import csv
 import logging
 import os
 import re
-from typing import Callable, Union
+from typing import Any, Dict, List, Optional, Union
 
 import simplejson as json
 
@@ -28,11 +28,17 @@ class MVTModule:
     """This class provides a base for all extraction modules."""
 
     enabled = True
-    slug = None
+    slug: Optional[str] = None
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = None, results: list = None):
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Union[List[Dict[str, Any]], Dict[str, Any], None] = None
+    ) -> None:
         """Initialize module.
 
         :param file_path: Path to the module's database file, if there is any
@@ -55,12 +61,12 @@ class MVTModule:
         self.log = log
         self.indicators = None
         self.results = results if results else []
-        self.detected = []
-        self.timeline = []
-        self.timeline_detected = []
+        self.detected: List[Dict[str, Any]] = []
+        self.timeline: List[Dict[str, str]] = []
+        self.timeline_detected: List[Dict[str, str]] = []
 
     @classmethod
-    def from_json(cls, json_path: str, log: logging.Logger = None):
+    def from_json(cls, json_path: str, log: logging.Logger):
         with open(json_path, "r", encoding="utf-8") as handle:
             results = json.load(handle)
             if log:
@@ -99,9 +105,9 @@ class MVTModule:
                 try:
                     json.dump(self.results, handle, indent=4, default=str)
                 except Exception as exc:
-                    self.log.error("Unable to store results of module %s "
-                                   "to file %s: %s", self.__class__.__name__,
-                                   results_file_name, exc)
+                    self.log.error("Unable to store results of module %s to file %s: %s",
+                                   self.__class__.__name__, results_file_name,
+                                   exc)
 
         if self.detected:
             detected_file_name = f"{name}_detected.json"
@@ -110,7 +116,7 @@ class MVTModule:
             with open(detected_json_path, "w", encoding="utf-8") as handle:
                 json.dump(self.detected, handle, indent=4, default=str)
 
-    def serialize(self, record: dict) -> Union[dict, list]:
+    def serialize(self, record: dict) -> Union[dict, list, None]:
         raise NotImplementedError
 
     @staticmethod
@@ -145,21 +151,22 @@ class MVTModule:
 
         # De-duplicate timeline entries.
         self.timeline = self._deduplicate_timeline(self.timeline)
-        self.timeline_detected = self._deduplicate_timeline(self.timeline_detected)
+        self.timeline_detected = self._deduplicate_timeline(
+            self.timeline_detected)
 
     def run(self) -> None:
         """Run the main module procedure."""
         raise NotImplementedError
 
 
-def run_module(module: Callable) -> None:
+def run_module(module: MVTModule) -> None:
     module.log.info("Running module %s...", module.__class__.__name__)
 
     try:
         module.run()
     except NotImplementedError:
-        module.log.exception("The run() procedure of module %s was not "
-                             "implemented yet!", module.__class__.__name__)
+        module.log.exception("The run() procedure of module %s was not implemented yet!",
+                             module.__class__.__name__)
     except InsufficientPrivileges as exc:
         module.log.info("Insufficient privileges for module %s: %s",
                         module.__class__.__name__, exc)
@@ -176,8 +183,12 @@ def run_module(module: Callable) -> None:
         try:
             module.check_indicators()
         except NotImplementedError:
-            module.log.info("The %s module does not support checking for "
-                            "indicators", module.__class__.__name__)
+            module.log.info("The %s module does not support checking for indicators",
+                            module.__class__.__name__)
+        except Exception as exc:
+            module.log.exception("Error when checking indicators from module %s: %s",
+                                 module.__class__.__name__, exc)
+
         else:
             if module.indicators and not module.detected:
                 module.log.info("The %s module produced no detections!",

mvt/common/options.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -16,18 +16,16 @@ class MutuallyExclusiveOption(Option):
         help_msg = kwargs.get("help", "")
         if self.mutually_exclusive:
             ex_str = ", ".join(self.mutually_exclusive)
-            kwargs["help"] = help_msg + (
-                " NOTE: This argument is mutually exclusive with "
-                "arguments: [" + ex_str + "]."
-            )
+            kwargs["help"] = (f"{help_msg} NOTE: This argument is mutually exclusive with arguments"
+                              f"[{ex_str}].")
 
         super().__init__(*args, **kwargs)
 
     def handle_parse_result(self, ctx, opts, args):
         if self.mutually_exclusive.intersection(opts) and self.name in opts:
             raise UsageError(
-                f"Illegal usage: `{self.name}` is mutually exclusive with "
-                f"arguments `{', '.join(self.mutually_exclusive)}`."
+                f"Illegal usage: `{self.name}` is mutually exclusive "
+                f"with arguments `{', '.join(self.mutually_exclusive)}`."
             )
 
         return super().handle_parse_result(ctx, opts, args)

mvt/common/updates.py

@@ -1,11 +1,12 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
 from datetime import datetime
+from typing import Optional, Tuple
 
 import requests
 import yaml
@@ -83,18 +84,18 @@ class IndicatorsUpdates:
         with open(self.latest_update_path, "w", encoding="utf-8") as handle:
             handle.write(str(timestamp))
 
-    def get_remote_index(self) -> dict:
+    def get_remote_index(self) -> Optional[dict]:
         url = self.github_raw_url.format(self.index_owner, self.index_repo,
                                          self.index_branch, self.index_path)
         res = requests.get(url)
         if res.status_code != 200:
-            log.error("Failed to retrieve indicators index located at %s "
-                      "(error %d)", url, res.status_code)
+            log.error("Failed to retrieve indicators index located at %s (error %d)",
+                      url, res.status_code)
             return None
 
         return yaml.safe_load(res.content)
 
-    def download_remote_ioc(self, ioc_url: str) -> str:
+    def download_remote_ioc(self, ioc_url: str) -> Optional[str]:
         res = requests.get(ioc_url)
         if res.status_code != 200:
             log.error("Failed to download indicators file from %s (error %d)",
@@ -116,6 +117,9 @@ class IndicatorsUpdates:
             os.makedirs(MVT_INDICATORS_FOLDER)
 
         index = self.get_remote_index()
+        if not index:
+            return
+
         for ioc in index.get("indicators", []):
             ioc_type = ioc.get("type", "")
 
@@ -131,8 +135,8 @@ class IndicatorsUpdates:
                 ioc_url = ioc.get("download_url", "")
 
             if not ioc_url:
-                log.error("Could not find a way to download indicator file "
-                          "for %s", ioc.get("name"))
+                log.error("Could not find a way to download indicator file for %s",
+                          ioc.get("name"))
                 continue
 
             ioc_local_path = self.download_remote_ioc(ioc_url)
@@ -162,8 +166,7 @@ class IndicatorsUpdates:
         latest_commit = details[0]
         latest_commit_date = latest_commit.get("commit", {}).get("author", {}).get("date", None)
         if not latest_commit_date:
-            log.error("Failed to retrieve date of latest update to indicators "
-                      "index file")
+            log.error("Failed to retrieve date of latest update to indicators index file")
             return -1
 
         latest_commit_dt = datetime.strptime(latest_commit_date,
@@ -172,7 +175,7 @@ class IndicatorsUpdates:
 
         return latest_commit_ts
 
-    def should_check(self) -> (bool, int):
+    def should_check(self) -> Tuple[bool, int]:
         now = datetime.utcnow()
         latest_check_ts = self.get_latest_check()
         latest_check_dt = datetime.fromtimestamp(latest_check_ts)
@@ -183,7 +186,7 @@ class IndicatorsUpdates:
         if diff_hours >= INDICATORS_CHECK_FREQUENCY:
             return True, 0
 
-        return False, INDICATORS_CHECK_FREQUENCY - diff_hours
+        return False, int(INDICATORS_CHECK_FREQUENCY - diff_hours)
 
     def check(self) -> bool:
         self.set_latest_check()
@@ -198,6 +201,9 @@ class IndicatorsUpdates:
             return True
 
         index = self.get_remote_index()
+        if not index:
+            return False
+
         for ioc in index.get("indicators", []):
             if ioc.get("type", "") != "github":
                 continue

mvt/common/url.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -264,7 +264,7 @@ class URL:
         self.top_level = self.get_top_level()
         self.is_shortened = False
 
-    def get_domain(self) -> None:
+    def get_domain(self) -> str:
         """Get the domain from a URL.
 
         :param url: URL to parse
@@ -273,15 +273,11 @@ class URL:
         :rtype: str
 
         """
-        # TODO: Properly handle exception.
-        try:
-            return get_tld(self.url,
-                           as_object=True,
-                           fix_protocol=True).parsed_url.netloc.lower().lstrip("www.")
-        except Exception:
-            return None
+        return get_tld(self.url,
+                       as_object=True,
+                       fix_protocol=True).parsed_url.netloc.lower().lstrip("www.")
 
-    def get_top_level(self) -> None:
+    def get_top_level(self) -> str:
         """Get only the top-level domain from a URL.
 
         :param url: URL to parse
@@ -290,11 +286,9 @@ class URL:
         :rtype: str
 
         """
-        # TODO: Properly handle exception.
-        try:
-            return get_tld(self.url, as_object=True, fix_protocol=True).fld.lower()
-        except Exception:
-            return None
+        return get_tld(self.url,
+                       as_object=True,
+                       fix_protocol=True).fld.lower()
 
     def check_if_shortened(self) -> bool:
         """Check if the URL is among list of shortener services.

mvt/common/utils.py

@@ -1,15 +1,16 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import datetime
 import hashlib
+import os
 import re
-from typing import Union
+from typing import Any, Iterator, Union
 
 
-def convert_chrometime_to_datetime(timestamp: int) -> int:
+def convert_chrometime_to_datetime(timestamp: int) -> datetime.datetime:
     """Converts Chrome timestamp to a datetime.
 
     :param timestamp: Chrome timestamp as int.
@@ -22,7 +23,7 @@ def convert_chrometime_to_datetime(timestamp: int) -> int:
     return epoch_start + delta
 
 
-def convert_datetime_to_iso(datetime: datetime.datetime) -> str:
+def convert_datetime_to_iso(date_time: datetime.datetime) -> str:
     """Converts datetime to ISO string.
 
     :param datetime: datetime.
@@ -32,12 +33,14 @@ def convert_datetime_to_iso(datetime: datetime.datetime) -> str:
 
     """
     try:
-        return datetime.strftime("%Y-%m-%d %H:%M:%S.%f")
+        return date_time.strftime("%Y-%m-%d %H:%M:%S.%f")
     except Exception:
         return ""
 
 
-def convert_unix_to_utc_datetime(timestamp: Union[int, float, str]) -> datetime.datetime:
+def convert_unix_to_utc_datetime(
+        timestamp: Union[int, float, str]
+) -> datetime.datetime:
     """Converts a unix epoch timestamp to UTC datetime.
 
     :param timestamp: Epoc timestamp to convert.
@@ -48,7 +51,7 @@ def convert_unix_to_utc_datetime(timestamp: Union[int, float, str]) -> datetime.
     return datetime.datetime.utcfromtimestamp(float(timestamp))
 
 
-def convert_unix_to_iso(timestamp: int) -> str:
+def convert_unix_to_iso(timestamp: Union[int, float, str]) -> str:
     """Converts a unix epoch to ISO string.
 
     :param timestamp: Epoc timestamp to convert.
@@ -105,8 +108,8 @@ def convert_mactime_to_iso(timestamp: int, from_2001: bool = True):
 
     """
 
-    return convert_datetime_to_iso(convert_mactime_to_datetime(timestamp,
-                                                               from_2001))
+    return convert_datetime_to_iso(
+        convert_mactime_to_datetime(timestamp, from_2001))
 
 
 def check_for_links(text: str) -> list:
@@ -120,24 +123,9 @@ def check_for_links(text: str) -> list:
     return re.findall(r"(?P<url>https?://[^\s]+)", text, re.IGNORECASE)
 
 
-def get_sha256_from_file_path(file_path: str) -> str:
-    """Calculate the SHA256 hash of a file from a file path.
-
-    :param file_path: Path to the file to hash
-    :returns: The SHA256 hash string
-
-    """
-    sha256_hash = hashlib.sha256()
-    with open(file_path, "rb") as handle:
-        for byte_block in iter(lambda: handle.read(4096), b""):
-            sha256_hash.update(byte_block)
-
-    return sha256_hash.hexdigest()
-
-
 # Note: taken from here:
 # https://stackoverflow.com/questions/57014259/json-dumps-on-dictionary-with-bytes-for-keys
-def keys_bytes_to_string(obj) -> str:
+def keys_bytes_to_string(obj: Any) -> Any:
     """Convert object keys from bytes to string.
 
     :param obj: Object to convert from bytes to string.
@@ -163,3 +151,49 @@ def keys_bytes_to_string(obj) -> str:
         new_obj[key] = value
 
     return new_obj
+
+
+def get_sha256_from_file_path(file_path: str) -> str:
+    """Calculate the SHA256 hash of a file from a file path.
+
+    :param file_path: Path to the file to hash
+    :returns: The SHA256 hash string
+
+    """
+    sha256_hash = hashlib.sha256()
+    try:
+        with open(file_path, "rb") as handle:
+            for byte_block in iter(lambda: handle.read(4096), b""):
+                sha256_hash.update(byte_block)
+    except OSError:
+        return ""
+
+    return sha256_hash.hexdigest()
+
+
+def generate_hashes_from_path(path: str, log) -> Iterator[dict]:
+    """
+    Generates hashes of all files at the given path.
+
+    :params path: Path of the given folder or file
+    :returns: generator of dict {"file_path", "hash"}
+    """
+    if os.path.isfile(path):
+        hash_value = get_sha256_from_file_path(path)
+        yield {"file_path": path, "sha256": hash_value}
+    elif os.path.isdir(path):
+        for (root, _, files) in os.walk(path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                try:
+                    sha256 = get_sha256_from_file_path(file_path)
+                except FileNotFoundError:
+                    log.error("Failed to hash the file %s: might be a symlink",
+                              file_path)
+                    continue
+                except PermissionError:
+                    log.error("Failed to hash the file %s: permission denied",
+                              file_path)
+                    continue
+
+                yield {"file_path": file_path, "sha256": sha256}

mvt/common/version.py

@@ -1,6 +1,6 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-MVT_VERSION = "2.1.4"
+MVT_VERSION = "2.2.4"

mvt/common/virustotal.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -42,8 +42,7 @@ def virustotal_lookup(file_hash: str):
     if res.status_code == 404:
         log.info("Could not find results for file with hash %s", file_hash)
     elif res.status_code == 429:
-        raise VTQuotaExceeded("You have exceeded the quota for your "
-                              "VirusTotal API key")
+        raise VTQuotaExceeded("You have exceeded the quota for your VirusTotal API key")
     else:
         raise Exception(f"Unexpected response from VirusTotal: {res.status_code}")
 

mvt/ios/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/ios/cli.py

@@ -1,8 +1,9 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import json
 import logging
 import os
 
@@ -11,12 +12,13 @@ from rich.logging import RichHandler
 from rich.prompt import Prompt
 
 from mvt.common.cmd_check_iocs import CmdCheckIOCS
-from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
+from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_HASHES, HELP_MSG_IOC,
                              HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
                              HELP_MSG_OUTPUT)
 from mvt.common.logo import logo
 from mvt.common.options import MutuallyExclusiveOption
 from mvt.common.updates import IndicatorsUpdates
+from mvt.common.utils import generate_hashes_from_path
 
 from .cmd_check_backup import CmdIOSCheckBackup
 from .cmd_check_fs import CmdIOSCheckFS
@@ -33,6 +35,7 @@ log = logging.getLogger(__name__)
 
 # Set this environment variable to a password if needed.
 MVT_IOS_BACKUP_PASSWORD = "MVT_IOS_BACKUP_PASSWORD"
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 
 #==============================================================================
@@ -54,7 +57,8 @@ def version():
 #==============================================================================
 # Command: decrypt-backup
 #==============================================================================
-@cli.command("decrypt-backup", help="Decrypt an encrypted iTunes backup")
+@cli.command("decrypt-backup", help="Decrypt an encrypted iTunes backup",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--destination", "-d", required=True,
               help="Path to the folder where to store the decrypted backup")
 @click.option("--password", "-p", cls=MutuallyExclusiveOption,
@@ -66,9 +70,10 @@ def version():
               help="File containing raw encryption key to use to decrypt "
                    "the backup",
               mutually_exclusive=["password"])
+@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
-def decrypt_backup(ctx, destination, password, key_file, backup_path):
+def decrypt_backup(ctx, destination, password, key_file, hashes, backup_path):
     backup = DecryptBackup(backup_path, destination)
 
     if key_file:
@@ -99,11 +104,22 @@ def decrypt_backup(ctx, destination, password, key_file, backup_path):
 
     backup.process_backup()
 
+    if hashes:
+        info = {"encrypted": [], "decrypted": []}
+        for file in generate_hashes_from_path(backup_path, log):
+            info["encrypted"].append(file)
+        for file in generate_hashes_from_path(destination, log):
+            info["decrypted"].append(file)
+        info_path = os.path.join(destination, "info.json")
+        with open(info_path, "w+", encoding="utf-8") as handle:
+            json.dump(info, handle, indent=4)
+
 
 #==============================================================================
 # Command: extract-key
 #==============================================================================
-@cli.command("extract-key", help="Extract decryption key from an iTunes backup")
+@cli.command("extract-key", help="Extract decryption key from an iTunes backup",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--password", "-p",
               help="Password to use to decrypt the backup (or, set "
                    f"{MVT_IOS_BACKUP_PASSWORD} environment variable)")
@@ -140,7 +156,8 @@ def extract_key(password, key_file, backup_path):
 #==============================================================================
 # Command: check-backup
 #==============================================================================
-@cli.command("check-backup", help="Extract artifacts from an iTunes backup")
+@cli.command("check-backup", help="Extract artifacts from an iTunes backup",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
 @click.option("--output", "-o", type=click.Path(exists=False),
@@ -148,11 +165,13 @@ def extract_key(password, key_file, backup_path):
 @click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST)
 @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
 @click.option("--module", "-m", help=HELP_MSG_MODULE)
+@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
-def check_backup(ctx, iocs, output, fast, list_modules, module, backup_path):
+def check_backup(ctx, iocs, output, fast, list_modules, module, hashes, backup_path):
     cmd = CmdIOSCheckBackup(target_path=backup_path, results_path=output,
-                            ioc_files=iocs, module_name=module, fast_mode=fast)
+                            ioc_files=iocs, module_name=module, fast_mode=fast,
+                            hashes=hashes)
 
     if list_modules:
         cmd.list_modules()
@@ -162,15 +181,16 @@ def check_backup(ctx, iocs, output, fast, list_modules, module, backup_path):
 
     cmd.run()
 
-    if len(cmd.timeline_detected) > 0:
+    if cmd.detected_count > 0:
         log.warning("The analysis of the backup produced %d detections!",
-                    len(cmd.timeline_detected))
+                    cmd.detected_count)
 
 
 #==============================================================================
 # Command: check-fs
 #==============================================================================
-@cli.command("check-fs", help="Extract artifacts from a full filesystem dump")
+@cli.command("check-fs", help="Extract artifacts from a full filesystem dump",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
 @click.option("--output", "-o", type=click.Path(exists=False),
@@ -178,11 +198,13 @@ def check_backup(ctx, iocs, output, fast, list_modules, module, backup_path):
 @click.option("--fast", "-f", is_flag=True, help=HELP_MSG_FAST)
 @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
 @click.option("--module", "-m", help=HELP_MSG_MODULE)
+@click.option("--hashes", "-H", is_flag=True, help=HELP_MSG_HASHES)
 @click.argument("DUMP_PATH", type=click.Path(exists=True))
 @click.pass_context
-def check_fs(ctx, iocs, output, fast, list_modules, module, dump_path):
+def check_fs(ctx, iocs, output, fast, list_modules, module, hashes, dump_path):
     cmd = CmdIOSCheckFS(target_path=dump_path, results_path=output,
-                        ioc_files=iocs, module_name=module, fast_mode=fast)
+                        ioc_files=iocs, module_name=module, fast_mode=fast,
+                        hashes=hashes)
 
     if list_modules:
         cmd.list_modules()
@@ -192,15 +214,16 @@ def check_fs(ctx, iocs, output, fast, list_modules, module, dump_path):
 
     cmd.run()
 
-    if len(cmd.timeline_detected) > 0:
+    if cmd.detected_count > 0:
         log.warning("The analysis of the iOS filesystem produced %d detections!",
-                    len(cmd.timeline_detected))
+                    cmd.detected_count)
 
 
 #==============================================================================
 # Command: check-iocs
 #==============================================================================
-@cli.command("check-iocs", help="Compare stored JSON results to provided indicators")
+@cli.command("check-iocs", help="Compare stored JSON results to provided indicators",
+             context_settings=CONTEXT_SETTINGS)
 @click.option("--iocs", "-i", type=click.Path(exists=True), multiple=True,
               default=[], help=HELP_MSG_IOC)
 @click.option("--list-modules", "-l", is_flag=True, help=HELP_MSG_LIST_MODULES)
@@ -221,7 +244,8 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
 #==============================================================================
 # Command: download-iocs
 #==============================================================================
-@cli.command("download-iocs", help="Download public STIX2 indicators")
+@cli.command("download-iocs", help="Download public STIX2 indicators",
+             context_settings=CONTEXT_SETTINGS)
 def download_iocs():
     ioc_updates = IndicatorsUpdates()
     ioc_updates.update()

mvt/ios/cmd_check_backup.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.common.command import Command
 
@@ -15,12 +16,20 @@ log = logging.getLogger(__name__)
 
 class CmdIOSCheckBackup(Command):
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        hashes: Optional[bool] = False,
+    ) -> None:
         super().__init__(target_path=target_path, results_path=results_path,
                          ioc_files=ioc_files, module_name=module_name,
-                         serial=serial, fast_mode=fast_mode, log=log)
+                         serial=serial, fast_mode=fast_mode, hashes=hashes,
+                         log=log)
 
         self.name = "check-backup"
         self.modules = BACKUP_MODULES + MIXED_MODULES

mvt/ios/cmd_check_fs.py

@@ -1,9 +1,10 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
+from typing import Optional
 
 from mvt.common.command import Command
 
@@ -15,12 +16,20 @@ log = logging.getLogger(__name__)
 
 class CmdIOSCheckFS(Command):
 
-    def __init__(self, target_path: str = None, results_path: str = None,
-                 ioc_files: list = [], module_name: str = None,
-                 serial: str = None, fast_mode: bool = False):
+    def __init__(
+        self,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        ioc_files: Optional[list] = None,
+        module_name: Optional[str] = None,
+        serial: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        hashes: Optional[bool] = False,
+    ) -> None:
         super().__init__(target_path=target_path, results_path=results_path,
                          ioc_files=ioc_files, module_name=module_name,
-                         serial=serial, fast_mode=fast_mode, log=log)
+                         serial=serial, fast_mode=fast_mode, hashes=hashes,
+                         log=log)
 
         self.name = "check-fs"
         self.modules = FS_MODULES + MIXED_MODULES

mvt/ios/decrypt.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -11,6 +11,7 @@ import os
 import os.path
 import shutil
 import sqlite3
+from typing import Optional
 
 from iOSbackup import iOSbackup
 
@@ -24,7 +25,7 @@ class DecryptBackup:
 
     """
 
-    def __init__(self, backup_path: str, dest_path: str = None) -> None:
+    def __init__(self, backup_path: str, dest_path: Optional[str] = None) -> None:
         """Decrypts an encrypted iOS backup.
         :param backup_path: Path to the encrypted backup folder
         :param dest_path: Path to the folder where to store the decrypted backup
@@ -93,8 +94,8 @@ class DecryptBackup:
                 if not os.path.exists(item_folder):
                     os.makedirs(item_folder)
 
-                # iOSBackup getFileDecryptedCopy() claims to read a "file" parameter
-                # but the code actually is reading the "manifest" key.
+                # iOSBackup getFileDecryptedCopy() claims to read a "file"
+                # parameter but the code actually is reading the "manifest" key.
                 # Add manifest plist to both keys to handle this.
                 item["manifest"] = item["file"]
 
@@ -111,7 +112,8 @@ class DecryptBackup:
         # Copying over the root plist files as well.
         for file_name in os.listdir(self.backup_path):
             if file_name.endswith(".plist"):
-                log.info("Copied plist file %s to %s", file_name, self.dest_path)
+                log.info("Copied plist file %s to %s",
+                         file_name, self.dest_path)
                 shutil.copy(os.path.join(self.backup_path, file_name),
                             self.dest_path)
 
@@ -121,18 +123,21 @@ class DecryptBackup:
         :param password: Password to use to decrypt the original backup
 
         """
-        log.info("Decrypting iOS backup at path %s with password", self.backup_path)
+        log.info("Decrypting iOS backup at path %s with password",
+                 self.backup_path)
 
         if not os.path.exists(os.path.join(self.backup_path, "Manifest.plist")):
-            possible = glob.glob(os.path.join(self.backup_path, "*", "Manifest.plist"))
+            possible = glob.glob(os.path.join(
+                self.backup_path, "*", "Manifest.plist"))
+
             if len(possible) == 1:
                 newpath = os.path.dirname(possible[0])
                 log.warning("No Manifest.plist in %s, using %s instead.",
                             self.backup_path, newpath)
                 self.backup_path = newpath
             elif len(possible) > 1:
-                log.critical("No Manifest.plist in %s, and %d Manifest.plist "
-                             "files in subdirs. Please choose one!",
+                log.critical("No Manifest.plist in %s, and %d Manifest.plist files in subdirs. "
+                             "Please choose one!",
                              self.backup_path, len(possible))
                 return
 
@@ -145,7 +150,9 @@ class DecryptBackup:
                                      cleartextpassword=password,
                                      backuproot=os.path.dirname(self.backup_path))
         except Exception as exc:
-            if isinstance(exc, KeyError) and len(exc.args) > 0 and exc.args[0] == b"KEY":
+            if (isinstance(exc, KeyError)
+                    and len(exc.args) > 0
+                    and exc.args[0] == b"KEY"):
                 log.critical("Failed to decrypt backup. Password is probably wrong.")
             elif (isinstance(exc, FileNotFoundError)
                     and os.path.basename(exc.filename) == "Manifest.plist"):
@@ -154,9 +161,8 @@ class DecryptBackup:
                              self.backup_path)
             else:
                 log.exception(exc)
-                log.critical("Failed to decrypt backup. Did you provide the "
-                             "correct password? Did you point to the right "
-                             "backup path?")
+                log.critical("Failed to decrypt backup. Did you provide the correct password? "
+                             "Did you point to the right backup path?")
 
     def decrypt_with_key_file(self, key_file: str) -> None:
         """Decrypts an encrypted iOS backup using a key file.
@@ -176,8 +182,7 @@ class DecryptBackup:
 
         # Key should be 64 hex encoded characters (32 raw bytes)
         if len(key_bytes) != 64:
-            log.critical("Invalid key from key file. Did you provide the "
-                         "correct key file?")
+            log.critical("Invalid key from key file. Did you provide the correct key file?")
             return
 
         try:
@@ -187,8 +192,7 @@ class DecryptBackup:
                                      backuproot=os.path.dirname(self.backup_path))
         except Exception as exc:
             log.exception(exc)
-            log.critical("Failed to decrypt backup. Did you provide the "
-                         "correct key file?")
+            log.critical("Failed to decrypt backup. Did you provide the correct key file?")
 
     def get_key(self) -> None:
         """Retrieve and prints the encryption key."""

mvt/ios/modules/__init__.py

@@ -1,4 +1,4 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/

mvt/ios/modules/backup/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/ios/modules/backup/backup_info.py

@@ -1,14 +1,15 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import os
 import plistlib
+from typing import Optional
 
 from mvt.common.module import DatabaseNotFoundError
-from mvt.ios.versions import get_device_desc_from_id, latest_ios_version
+from mvt.ios.versions import get_device_desc_from_id, is_ios_version_outdated
 
 from ..base import IOSExtraction
 
@@ -16,10 +17,15 @@ from ..base import IOSExtraction
 class BackupInfo(IOSExtraction):
     """This module extracts information about the device and the backup."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -29,8 +35,8 @@ class BackupInfo(IOSExtraction):
     def run(self) -> None:
         info_path = os.path.join(self.target_path, "Info.plist")
         if not os.path.exists(info_path):
-            raise DatabaseNotFoundError("No Info.plist at backup path, unable "
-                                        "to extract device information")
+            raise DatabaseNotFoundError("No Info.plist at backup path, unable to extract device "
+                                        "information")
 
         with open(info_path, "rb") as handle:
             info = plistlib.load(handle)
@@ -44,7 +50,7 @@ class BackupInfo(IOSExtraction):
 
         for field in fields:
             value = info.get(field, None)
-            # Converting the product type in product name
+
             if field == "Product Type" and value:
                 product_name = get_device_desc_from_id(value)
                 if product_name:
@@ -53,11 +59,8 @@ class BackupInfo(IOSExtraction):
                     self.log.info("%s: %s", field, value)
             else:
                 self.log.info("%s: %s", field, value)
+
             self.results[field] = value
 
         if "Product Version" in info:
-            latest = latest_ios_version()
-            if info["Product Version"] != latest["version"]:
-                self.log.warning("This phone is running an outdated iOS "
-                                 "version: %s (latest is %s)",
-                                 info["Product Version"], latest['version'])
+            is_ios_version_outdated(info["Product Version"], log=self.log)

mvt/ios/modules/backup/configuration_profiles.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -7,7 +7,7 @@ import logging
 import os
 import plistlib
 from base64 import b64encode
-from typing import Union
+from typing import Optional, Union
 
 from mvt.common.utils import convert_datetime_to_iso
 
@@ -19,10 +19,15 @@ CONF_PROFILES_DOMAIN = "SysSharedContainerDomain-systemgroup.com.apple.configura
 class ConfigurationProfiles(IOSExtraction):
     """This module extracts the full plist data from configuration profiles."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -37,9 +42,8 @@ class ConfigurationProfiles(IOSExtraction):
             "timestamp": record["install_date"],
             "module": self.__class__.__name__,
             "event": "configuration_profile_install",
-            "data": f"{record['plist']['PayloadType']} installed: "
-                    f"{record['plist']['PayloadUUID']} - "
-                    f"{payload_name}: {payload_description}"
+            "data": f"{record['plist']['PayloadType']} installed: {record['plist']['PayloadUUID']} "
+                    f"- {payload_name}: {payload_description}"
         }
 
     def check_indicators(self) -> None:
@@ -54,9 +58,10 @@ class ConfigurationProfiles(IOSExtraction):
                 # indicator list.
                 ioc = self.indicators.check_profile(result["plist"]["PayloadUUID"])
                 if ioc:
-                    self.log.warning(f"Found a known malicious configuration profile "
-                                     f"\"{result['plist']['PayloadDisplayName']}\" "
-                                     f"with UUID '{result['plist']['PayloadUUID']}'.")
+                    self.log.warning("Found a known malicious configuration "
+                                     "profile \"%s\" with UUID %s",
+                                     result['plist']['PayloadDisplayName'],
+                                     result['plist']['PayloadUUID'])
                     result["matched_indicator"] = ioc
                     self.detected.append(result)
                     continue
@@ -64,19 +69,22 @@ class ConfigurationProfiles(IOSExtraction):
                 # Highlight suspicious configuration profiles which may be used
                 # to hide notifications.
                 if payload_content["PayloadType"] in ["com.apple.notificationsettings"]:
-                    self.log.warning(f"Found a potentially suspicious configuration profile "
-                                     f"\"{result['plist']['PayloadDisplayName']}\" with "
-                                     f"payload type '{payload_content['PayloadType']}'.")
+                    self.log.warning("Found a potentially suspicious configuration profile "
+                                     "\"%s\" with payload type %s",
+                                     result['plist']['PayloadDisplayName'],
+                                     payload_content['PayloadType'])
                     self.detected.append(result)
                     continue
 
     def run(self) -> None:
-        for conf_file in self._get_backup_files_from_manifest(domain=CONF_PROFILES_DOMAIN):
+        for conf_file in self._get_backup_files_from_manifest(
+                domain=CONF_PROFILES_DOMAIN):
             conf_rel_path = conf_file["relative_path"]
 
             # Filter out all configuration files that are not configuration
             # profiles.
-            if not conf_rel_path or not os.path.basename(conf_rel_path).startswith("profile-"):
+            if not conf_rel_path or not os.path.basename(
+                    conf_rel_path).startswith("profile-"):
                 continue
 
             conf_file_path = self._get_backup_file_from_id(conf_file["file_id"])
@@ -89,6 +97,8 @@ class ConfigurationProfiles(IOSExtraction):
                 except Exception:
                     conf_plist = {}
 
+            # TODO: Tidy up the following code hell.
+
             if "SignerCerts" in conf_plist:
                 conf_plist["SignerCerts"] = [b64encode(x) for x in conf_plist["SignerCerts"]]
 
@@ -122,4 +132,5 @@ class ConfigurationProfiles(IOSExtraction):
                 "install_date": convert_datetime_to_iso(conf_plist.get("InstallDate")),
             })
 
-        self.log.info("Extracted details about %d configuration profiles", len(self.results))
+        self.log.info("Extracted details about %d configuration profiles",
+                      len(self.results))

mvt/ios/modules/backup/manifest.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -9,8 +9,10 @@ import logging
 import os
 import plistlib
 import sqlite3
+from typing import Optional
 
 from mvt.common.module import DatabaseNotFoundError
+from mvt.common.url import URL
 from mvt.common.utils import convert_datetime_to_iso, convert_unix_to_iso
 
 from ..base import IOSExtraction
@@ -19,10 +21,15 @@ from ..base import IOSExtraction
 class Manifest(IOSExtraction):
     """This module extracts information from a backup Manifest.db file."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -73,9 +80,6 @@ class Manifest(IOSExtraction):
         return records
 
     def check_indicators(self) -> None:
-        if not self.indicators:
-            return
-
         for result in self.results:
             if not result.get("relative_path"):
                 continue
@@ -84,21 +88,30 @@ class Manifest(IOSExtraction):
                 if (os.path.basename(result["relative_path"]) == "com.apple.CrashReporter.plist"
                         and result["domain"] == "RootDomain"):
                     self.log.warning("Found a potentially suspicious "
-                                     "\"com.apple.CrashReporter.plist\" "
-                                     "file created in RootDomain")
+                                     "\"com.apple.CrashReporter.plist\" file created in RootDomain")
                     self.detected.append(result)
                     continue
 
+            if not self.indicators:
+                continue
+
             if self.indicators.check_file_path("/" + result["relative_path"]):
                 self.detected.append(result)
                 continue
 
             rel_path = result["relative_path"].lower()
-            for ioc in self.indicators.get_iocs("domains"):
-                if ioc["value"].lower() in rel_path:
-                    self.log.warning("Found mention of domain \"%s\" in a "
-                                     "backup file with path: %s",
-                                     ioc["value"], rel_path)
+            parts = rel_path.split("_")
+            for part in parts:
+                try:
+                    URL(part)
+                except Exception:
+                    continue
+
+                ioc = self.indicators.check_domain(part)
+                if ioc:
+                    self.log.warning("Found mention of domain \"%s\" in a backup file with "
+                                     "path: %s", ioc["value"], rel_path)
+                    result["matched_indicator"] = ioc
                     self.detected.append(result)
 
     def run(self) -> None:
@@ -132,19 +145,23 @@ class Manifest(IOSExtraction):
                 try:
                     file_plist = plistlib.load(io.BytesIO(file_data["file"]))
                     file_metadata = self._get_key(file_plist, "$objects")[1]
+
+                    birth = self._get_key(file_metadata, "Birth")
+                    last_modified = self._get_key(file_metadata, "LastModified")
+                    last_status_change = self._get_key(file_metadata,
+                                                       "LastStatusChange")
+
                     cleaned_metadata.update({
-                        "created": self._convert_timestamp(self._get_key(file_metadata, "Birth")),
-                        "modified": self._convert_timestamp(self._get_key(file_metadata,
-                                                                          "LastModified")),
-                        "status_changed": self._convert_timestamp(self._get_key(file_metadata,
-                                                                                "LastStatusChange")),
+                        "created": self._convert_timestamp(birth),
+                        "modified": self._convert_timestamp(last_modified),
+                        "status_changed": self._convert_timestamp(last_status_change),
                         "mode": oct(self._get_key(file_metadata, "Mode")),
                         "owner": self._get_key(file_metadata, "UserID"),
                         "size": self._get_key(file_metadata, "Size"),
                     })
                 except Exception:
-                    self.log.exception("Error reading manifest file metadata "
-                                       "for file with ID %s and relative path %s",
+                    self.log.exception("Error reading manifest file metadata for file with ID %s "
+                                       "and relative path %s",
                                        file_data["fileID"],
                                        file_data["relativePath"])
 

mvt/ios/modules/backup/profile_events.py

@@ -1,16 +1,17 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
 import logging
 import plistlib
-from typing import Union
+from typing import Optional, Union
 
 from mvt.common.utils import convert_datetime_to_iso
 
 from ..base import IOSExtraction
 
+# CONF_PROFILES_EVENTS_ID = "aeb25de285ea542f7ac7c2070cddd1961e369df1"
 CONF_PROFILES_EVENTS_RELPATH = "Library/ConfigurationProfiles/MCProfileEvents.plist"
 
 
@@ -20,10 +21,15 @@ class ProfileEvents(IOSExtraction):
 
 
     """
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -85,8 +91,10 @@ class ProfileEvents(IOSExtraction):
         return results
 
     def run(self) -> None:
-        for events_file in self._get_backup_files_from_manifest(relative_path=CONF_PROFILES_EVENTS_RELPATH):
-            events_file_path = self._get_backup_file_from_id(events_file["file_id"])
+        for events_file in self._get_backup_files_from_manifest(
+                relative_path=CONF_PROFILES_EVENTS_RELPATH):
+            events_file_path = self._get_backup_file_from_id(
+                events_file["file_id"])
             if not events_file_path:
                 continue
 
@@ -97,8 +105,7 @@ class ProfileEvents(IOSExtraction):
                 self.results.extend(self.parse_profile_events(handle.read()))
 
         for result in self.results:
-            self.log.info("On %s process \"%s\" started operation \"%s\" "
-                          "of profile \"%s\"",
+            self.log.info("On %s process \"%s\" started operation \"%s\" of profile \"%s\"",
                           result.get("timestamp"), result.get("process"),
                           result.get("operation"), result.get("profile_id"))
 

mvt/ios/modules/base.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
@@ -9,6 +9,7 @@ import os
 import shutil
 import sqlite3
 import subprocess
+from typing import Iterator, Optional, Union
 
 from mvt.common.module import (DatabaseCorruptedError, DatabaseNotFoundError,
                                MVTModule)
@@ -18,19 +19,24 @@ class IOSExtraction(MVTModule):
     """This class provides a base for all iOS filesystem/backup extraction
     modules."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
 
         self.is_backup = False
         self.is_fs_dump = False
-        self.is_sysdiagnose = False
 
-    def _recover_sqlite_db_if_needed(self, file_path, forced=False):
+    def _recover_sqlite_db_if_needed(self, file_path: str,
+                                     forced: Optional[bool] = False) -> None:
         """Tries to recover a malformed database by running a .clone command.
 
         :param file_path: Path to the malformed database file.
@@ -57,13 +63,11 @@ class IOSExtraction(MVTModule):
                       file_path)
 
         if not shutil.which("sqlite3"):
-            raise DatabaseCorruptedError("failed to recover without sqlite3 "
-                                         "binary: please install sqlite3!")
+            raise DatabaseCorruptedError("failed to recover without sqlite3 binary: please install "
+                                         "sqlite3!")
         if '"' in file_path:
-            raise DatabaseCorruptedError(f"database at path '{file_path}' is "
-                                         "corrupted. unable to recover because "
-                                         "it has a quotation mark (\") in its "
-                                         "name")
+            raise DatabaseCorruptedError(f"database at path '{file_path}' is corrupted. unable to "
+                                         "recover because it has a quotation mark (\") in its name")
 
         bak_path = f"{file_path}.bak"
         shutil.move(file_path, bak_path)
@@ -75,7 +79,11 @@ class IOSExtraction(MVTModule):
 
         self.log.info("Database at path %s recovered successfully!", file_path)
 
-    def _get_backup_files_from_manifest(self, relative_path=None, domain=None):
+    def _get_backup_files_from_manifest(
+        self,
+        relative_path: Optional[str] = None,
+        domain: Optional[str] = None
+    ) -> Iterator[dict]:
         """Locate files from Manifest.db.
 
         :param relative_path: Relative path to use as filter from Manifest.db.
@@ -98,8 +106,12 @@ class IOSExtraction(MVTModule):
                             (relative_path, domain))
             else:
                 if relative_path:
-                    cur.execute(f"{base_sql} relativePath = ?;",
-                                (relative_path,))
+                    if "*" in relative_path:
+                        cur.execute(f"{base_sql} relativePath LIKE ?;",
+                                    (relative_path.replace("*", "%"),))
+                    else:
+                        cur.execute(f"{base_sql} relativePath = ?;",
+                                    (relative_path,))
                 elif domain:
                     cur.execute(f"{base_sql} domain = ?;", (domain,))
         except Exception as exc:
@@ -112,14 +124,14 @@ class IOSExtraction(MVTModule):
                 "relative_path": row[2],
             }
 
-    def _get_backup_file_from_id(self, file_id):
+    def _get_backup_file_from_id(self, file_id: str) -> Union[str, None]:
         file_path = os.path.join(self.target_path, file_id[0:2], file_id)
         if os.path.exists(file_path):
             return file_path
 
         return None
 
-    def _get_fs_files_from_patterns(self, root_paths):
+    def _get_fs_files_from_patterns(self, root_paths: list) -> Iterator[str]:
         for root_path in root_paths:
             for found_path in glob.glob(os.path.join(self.target_path,
                                                      root_path)):
@@ -128,14 +140,17 @@ class IOSExtraction(MVTModule):
 
                 yield found_path
 
-    def _find_ios_database(self, backup_ids=None, root_paths=[]):
+    def _find_ios_database(
+        self,
+        backup_ids: Optional[list] = None,
+        root_paths: Optional[list] = None
+    ) -> None:
         """Try to locate a module's database file from either an iTunes
         backup or a full filesystem dump. This is intended only for
         modules that expect to work with a single SQLite database.
         If a module requires to process multiple databases or files,
         you should use the helper functions above.
 
-        :param backup_id: iTunes backup database file's ID (or hash).
         :param root_paths: Glob patterns for files to seek in filesystem dump.
                            (Default value = [])
         :param backup_ids: Default value = None)
@@ -153,20 +168,20 @@ class IOSExtraction(MVTModule):
                     if file_path:
                         break
 
-            # If this file does not exist we might be processing a full
-            # filesystem dump (checkra1n all the things!).
-            if not file_path or not os.path.exists(file_path):
-                # We reset the file_path.
-                file_path = None
-                for found_path in self._get_fs_files_from_patterns(root_paths):
-                    file_path = found_path
-                    break
+            if root_paths:
+                # If this file does not exist we might be processing a full
+                # filesystem dump (checkra1n all the things!).
+                if not file_path or not os.path.exists(file_path):
+                    # We reset the file_path.
+                    file_path = None
+                    for found_path in self._get_fs_files_from_patterns(root_paths):
+                        file_path = found_path
+                        break
 
         # If we do not find any, we fail.
         if file_path:
             self.file_path = file_path
         else:
-            raise DatabaseNotFoundError("unable to find the module's "
-                                        "database file")
+            raise DatabaseNotFoundError("unable to find the module's database file")
 
         self._recover_sqlite_db_if_needed(self.file_path)

mvt/ios/modules/fs/__init__.py

@@ -1,5 +1,5 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 

mvt/ios/modules/fs/analytics.py

@@ -1,12 +1,13 @@
 # Mobile Verification Toolkit (MVT)
-# Copyright (c) 2021-2022 Claudio Guarnieri.
+# Copyright (c) 2021-2023 Claudio Guarnieri.
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import copy
 import logging
 import plistlib
 import sqlite3
-from typing import Union
+from typing import Optional, Union
 
 from mvt.common.utils import convert_mactime_to_iso
 
@@ -21,10 +22,15 @@ class Analytics(IOSExtraction):
     """This module extracts information from the
     private/var/Keychains/Analytics/*.db files."""
 
-    def __init__(self, file_path: str = None, target_path: str = None,
-                 results_path: str = None, fast_mode: bool = False,
-                 log: logging.Logger = logging.getLogger(__name__),
-                 results: list = []) -> None:
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        fast_mode: Optional[bool] = False,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None
+    ) -> None:
         super().__init__(file_path=file_path, target_path=target_path,
                          results_path=results_path, fast_mode=fast_mode,
                          log=log, results=results)
@@ -48,22 +54,22 @@ class Analytics(IOSExtraction):
 
                 ioc = self.indicators.check_process(value)
                 if ioc:
-                    self.log.warning("Found mention of a malicious process "
-                                     "\"%s\" in %s file at %s",
+                    self.log.warning("Found mention of a malicious process \"%s\" in %s file at %s",
                                      value, result["artifact"],
-                                     result["timestamp"])
-                    result["matched_indicator"] = ioc
-                    self.detected.append(result)
+                                     result["isodate"])
+                    new_result = copy.copy(result)
+                    new_result["matched_indicator"] = ioc
+                    self.detected.append(new_result)
                     continue
 
                 ioc = self.indicators.check_domain(value)
                 if ioc:
-                    self.log.warning("Found mention of a malicious domain "
-                                     "\"%s\" in %s file at %s",
+                    self.log.warning("Found mention of a malicious domain \"%s\" in %s file at %s",
                                      value, result["artifact"],
-                                     result["timestamp"])
-                    result["matched_indicator"] = ioc
-                    self.detected.append(result)
+                                     result["isodate"])
+                    new_result = copy.copy(result)
+                    new_result["matched_indicator"] = ioc
+                    self.detected.append(new_result)
 
     def _extract_analytics_data(self):
         artifact = self.file_path.split("/")[-1]