Bumped version

Install libimobiledevice-glue from source

Dockerfile

@@ -38,12 +38,15 @@ RUN apt update \
 # Build libimobiledevice
 # ----------------------
 RUN git clone https://github.com/libimobiledevice/libplist \
+  && git clone https://github.com/libimobiledevice/libimobiledevice-glue \
   && git clone https://github.com/libimobiledevice/libusbmuxd \
   && git clone https://github.com/libimobiledevice/libimobiledevice \
   && git clone https://github.com/libimobiledevice/usbmuxd \
 
   && cd libplist && ./autogen.sh && make && make install && ldconfig \
 
+  && cd ../libimobiledevice-glue && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr && make && make install && ldconfig \
+
   && cd ../libusbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh && make && make install && ldconfig \
 
   && cd ../libimobiledevice && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --enable-debug && make && make install && ldconfig \
@@ -51,7 +54,7 @@ RUN git clone https://github.com/libimobiledevice/libplist \
   && cd ../usbmuxd && PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --runstatedir=/run && make && make install \
 
   # Clean up.
-  && cd .. && rm -rf libplist libusbmuxd libimobiledevice usbmuxd
+  && cd .. && rm -rf libplist libimobiledevice-glue libusbmuxd libimobiledevice usbmuxd
 
 # Installing MVT
 # --------------

docs/android/backup.md

@@ -22,7 +22,7 @@ adb backup -all
 
 ## Unpack the backup
 
-In order to reliable unpack th [Android Backup Extractor (ABE)](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
+In order to unpack the backup, use [Android Backup Extractor (ABE)](https://github.com/nelenkov/android-backup-extractor) to convert it to a readable file format. Make sure that java is installed on your system and use the following command:
 
 ```bash
 java -jar ~/path/to/abe.jar unpack backup.ab backup.tar
@@ -31,6 +31,8 @@ tar xvf backup.tar
 
 If the backup is encrypted, ABE will prompt you to enter the password.
 
+Alternatively, [ab-decrypt](https://github.com/joernheissler/ab-decrypt) can be used for that purpose.
+
 ## Check the backup
 
 You can then extract SMSs containing links with MVT:

docs/iocs.md

@@ -32,5 +32,6 @@ mvt-ios check-backup --iocs ~/iocs/malware1.stix --iocs ~/iocs/malware2.stix2 /p
 
 - The [Amnesty International investigations repository](https://github.com/AmnestyTech/investigations) contains STIX-formatted IOCs for:
     - [Pegasus](https://en.wikipedia.org/wiki/Pegasus_(spyware)) ([STIX2](https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/pegasus.stix2))
+- [This repository](https://github.com/Te-k/stalkerware-indicators) contains IOCs for Android stalkerware including [a STIX MVT-compatible file](https://github.com/Te-k/stalkerware-indicators/blob/master/stalkerware.stix2).
 
 Please [open an issue](https://github.com/mvt-project/mvt/issues/) to suggest new sources of STIX-formatted IOCs.

docs/ios/records.md

@@ -230,6 +230,18 @@ If indicators are provided through the command-line, they are checked against th
 
 ---
 
+### `shutdown_log.json`
+
+!!! info "Availability"
+    Backup (if encrypted): :material-close:  
+    Full filesystem dump: :material-check:
+
+This JSON file is created by mvt-ios' `ShutdownLog` module. The module extracts records from the shutdown log located at *private/var/db/diagnostics/shutdown.log*. When shutting down an iPhone, a SIGTERM will be sent to all processes runnning. The `shutdown.log` file will log any process (with its pid and path) that did not shut down after the SIGTERM was sent. 
+
+If indicators are provided through the command-line, they are checked against the paths. Any matches are stored in *shutdown_log_detected.json*.
+
+---
+
 ### `sms.json`
 
 !!! info "Availability"

mvt/android/cli.py

@@ -34,6 +34,14 @@ def cli():
     logo()
 
 
+#==============================================================================
+# Command: version
+#==============================================================================
+@cli.command("version", help="Show the currently installed version of MVT")
+def version():
+    return
+
+
 #==============================================================================
 # Download APKs
 #==============================================================================

mvt/android/download_apks.py

@@ -11,7 +11,6 @@ import pkg_resources
 from tqdm import tqdm
 
 from mvt.common.module import InsufficientPrivileges
-from mvt.common.utils import get_sha256_from_file_path
 
 from .modules.adb.base import AndroidExtraction
 from .modules.adb.packages import Packages
@@ -158,37 +157,16 @@ class DownloadAPKs(AndroidExtraction):
             log.info("[%d/%d] Package: %s", counter, len(packages_selection),
                      package["package_name"])
 
-            # Get the file path for the specific package.
-            try:
-                output = self._adb_command(f"pm path {package['package_name']}")
-                output = output.strip().replace("package:", "")
-                if not output:
-                    continue
-            except Exception as e:
-                log.exception("Failed to get path of package %s: %s",
-                              package["package_name"], e)
-                self._adb_reconnect()
-                continue
-
             # Sometimes the package path contains multiple lines for multiple apks.
             # We loop through each line and download each file.
-            for path in output.split("\n"):
-                device_path = path.strip()
-                file_path = self.pull_package_file(package["package_name"],
-                                                   device_path)
-                if not file_path:
+            for package_file in package["files"]:
+                device_path = package_file["path"]
+                local_path = self.pull_package_file(package["package_name"],
+                                                    device_path)
+                if not local_path:
                     continue
 
-                file_info = {
-                    "path": device_path,
-                    "local_name": file_path,
-                    "sha256": get_sha256_from_file_path(file_path),
-                }
-
-                if "files" not in package:
-                    package["files"] = [file_info,]
-                else:
-                    package["files"].append(file_info)
+                package_file["local_path"] = local_path
 
         log.info("Download of selected packages completed")
 

mvt/android/lookups/koodous.py

@@ -32,7 +32,7 @@ def koodous_lookup(packages):
             res = requests.get(url)
             report = res.json()
 
-            row = [package["package_name"], file["local_name"]]
+            row = [package["package_name"], file["path"]]
 
             if "package_name" in report:
                 trusted = "no"

mvt/android/lookups/virustotal.py

@@ -75,7 +75,7 @@ def virustotal_lookup(packages):
 
     for package in packages:
         for file in package.get("files", []):
-            row = [package["package_name"], file["local_name"]]
+            row = [package["package_name"], file["path"]]
 
             if file["sha256"] in detections:
                 detection = detections[file["sha256"]]

mvt/android/modules/adb/base.py

@@ -41,7 +41,7 @@ class AndroidExtraction(MVTModule):
     def _adb_check_keys():
         """Make sure Android adb keys exist."""
         if not os.path.isdir(os.path.dirname(ADB_KEY_PATH)):
-            os.path.makedirs(os.path.dirname(ADB_KEY_PATH))
+            os.makedirs(os.path.dirname(ADB_KEY_PATH))
 
         if not os.path.exists(ADB_KEY_PATH):
             keygen(ADB_KEY_PATH)
@@ -56,7 +56,10 @@ class AndroidExtraction(MVTModule):
         with open(ADB_KEY_PATH, "rb") as handle:
             priv_key = handle.read()
 
-        signer = PythonRSASigner("", priv_key)
+        with open(ADB_PUB_KEY_PATH, "rb") as handle:
+            pub_key = handle.read()
+
+        signer = PythonRSASigner(pub_key, priv_key)
 
         # If no serial was specified or if the serial does not seem to be
         # a HOST:PORT definition, we use the USB transport.
@@ -132,7 +135,7 @@ class AndroidExtraction(MVTModule):
 
         """
         return self._adb_command(f"su -c {command}")
-    
+
     def _adb_check_file_exists(self, file):
         """Verify that a file exists.
 
@@ -166,7 +169,7 @@ class AndroidExtraction(MVTModule):
                 self._adb_download_root(remote_path, local_path, progress_callback)
             else:
                 raise Exception(f"Unable to download file {remote_path}: {e}")
-    
+
     def _adb_download_root(self, remote_path, local_path, progress_callback=None):
         try:
             # Check if we have root, if not raise an Exception.
@@ -191,7 +194,7 @@ class AndroidExtraction(MVTModule):
 
             # Delete the copy on /sdcard/.
             self._adb_command(f"rm -rf {new_remote_path}")
-            
+
         except AdbCommandFailureException as e:
             raise Exception(f"Unable to download file {remote_path}: {e}")
 

mvt/android/modules/adb/chrome_history.py

@@ -33,6 +33,14 @@ class ChromeHistory(AndroidExtraction):
             "data": f"{record['id']} - {record['url']} (visit ID: {record['visit_id']}, redirect source: {record['redirect_source']})"
         }
 
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            if self.indicators.check_domain(result["url"]):
+                self.detected.append(result)
+
     def _parse_db(self, db_path):
         """Parse a Chrome History database file.
 

mvt/android/modules/adb/packages.py

@@ -41,19 +41,55 @@ class Packages(AndroidExtraction):
         return records
 
     def check_indicators(self):
+        if not self.indicators:
+            return
+
         root_packages_path = os.path.join("..", "..", "data", "root_packages.txt")
         root_packages_string = pkg_resources.resource_string(__name__, root_packages_path)
         root_packages = root_packages_string.decode("utf-8").split("\n")
+        root_packages = [rp.strip() for rp in root_packages]
 
-        for root_package in root_packages:
-            root_package = root_package.strip()
-            if not root_package:
-                continue
 
-            if root_package in self.results:
+        for result in self.results:
+            if result["package_name"] in root_packages:
                 self.log.warning("Found an installed package related to rooting/jailbreaking: \"%s\"",
-                                 root_package)
-                self.detected.append(root_package)
+                                result["package_name"])
+                self.detected.append(result)
+            if result["package_name"] in self.indicators.ioc_app_ids:
+                self.log.warning("Found a malicious package name: \"%s\"",
+                                 result["package_name"])
+                self.detected.append(result)
+                for file in result["files"]:
+                    if file["sha256"] in self.indicators.ioc_files_sha256:
+                        self.log.warning("Found a malicious APK: \"%s\" %s",
+                                         result["package_name"],
+                                         file["sha256"])
+                        self.detected.append(result)
+
+    def _get_files_for_package(self, package_name):
+        output = self._adb_command(f"pm path {package_name}")
+        output = output.strip().replace("package:", "")
+        if not output:
+            return []
+
+        package_files = []
+        for file_path in output.split("\n"):
+            file_path = file_path.strip()
+
+            md5 = self._adb_command(f"md5sum {file_path}").split(" ")[0]
+            sha1 = self._adb_command(f"sha1sum {file_path}").split(" ")[0]
+            sha256 = self._adb_command(f"sha256sum {file_path}").split(" ")[0]
+            sha512 = self._adb_command(f"sha512sum {file_path}").split(" ")[0]
+
+            package_files.append({
+                "path": file_path,
+                "md5": md5,
+                "sha1": sha1,
+                "sha256": sha256,
+                "sha512": sha512,
+            })
+
+        return package_files
 
     def run(self):
         self._adb_connect()
@@ -85,6 +121,8 @@ class Packages(AndroidExtraction):
             first_install = dumpsys[1].split("=")[1].strip()
             last_update = dumpsys[2].split("=")[1].strip()
 
+            package_files = self._get_files_for_package(package_name)
+
             self.results.append({
                 "package_name": package_name,
                 "file_name": file_name,
@@ -96,6 +134,7 @@ class Packages(AndroidExtraction):
                 "disabled": False,
                 "system": False,
                 "third_party": False,
+                "files": package_files,
             })
 
         cmds = [

mvt/common/indicators.py

@@ -15,6 +15,8 @@ class IndicatorsFileBadFormat(Exception):
 class Indicators:
     """This class is used to parse indicators from a STIX2 file and provide
     functions to compare extracted artifacts to the indicators.
+
+
     """
 
     def __init__(self, log=None):
@@ -23,6 +25,8 @@ class Indicators:
         self.ioc_processes = []
         self.ioc_emails = []
         self.ioc_files = []
+        self.ioc_files_sha256 = []
+        self.ioc_app_ids = []
         self.ioc_count = 0
 
     def _add_indicator(self, ioc, iocs_list):
@@ -35,6 +39,7 @@ class Indicators:
 
         :param file_path: Path to the STIX2 file to parse
         :type file_path: str
+
         """
         self.log.info("Parsing STIX2 indicators file at path %s",
                       file_path)
@@ -66,6 +71,12 @@ class Indicators:
             elif key == "file:name":
                 self._add_indicator(ioc=value,
                                     iocs_list=self.ioc_files)
+            elif key == "app:id":
+                self._add_indicator(ioc=value,
+                                    iocs_list=self.ioc_app_ids)
+            elif key == "file:hashes.sha256":
+                self._add_indicator(ioc=value,
+                                    iocs_list=self.ioc_files_sha256)
 
     def check_domain(self, url) -> bool:
         """Check if a given URL matches any of the provided domain indicators.
@@ -74,6 +85,7 @@ class Indicators:
         :type url: str
         :returns: True if the URL matched an indicator, otherwise False
         :rtype: bool
+
         """
         # TODO: If the IOC domain contains a subdomain, it is not currently
         # being matched.
@@ -145,6 +157,7 @@ class Indicators:
         :type urls: list
         :returns: True if any URL matched an indicator, otherwise False
         :rtype: bool
+
         """
         if not urls:
             return False
@@ -163,6 +176,7 @@ class Indicators:
         :type process: str
         :returns: True if process matched an indicator, otherwise False
         :rtype: bool
+
         """
         if not process:
             return False
@@ -188,6 +202,7 @@ class Indicators:
         :type processes: list
         :returns: True if process matched an indicator, otherwise False
         :rtype: bool
+
         """
         if not processes:
             return False
@@ -205,6 +220,7 @@ class Indicators:
         :type email: str
         :returns: True if email address matched an indicator, otherwise False
         :rtype: bool
+
         """
         if not email:
             return False
@@ -223,6 +239,7 @@ class Indicators:
         :type file_path: str
         :returns: True if the file path matched an indicator, otherwise False
         :rtype: bool
+
         """
         if not file_path:
             return False

mvt/common/module.py

@@ -23,8 +23,7 @@ class InsufficientPrivileges(Exception):
     pass
 
 class MVTModule(object):
-    """This class provides a base for all extraction modules.
-    """
+    """This class provides a base for all extraction modules."""
 
     enabled = True
     slug = None
@@ -66,8 +65,7 @@ class MVTModule(object):
             return cls(results=results, log=log)
 
     def get_slug(self):
-        """Use the module's class name to retrieve a slug
-        """
+        """Use the module's class name to retrieve a slug"""
         if self.slug:
             return self.slug
 
@@ -77,12 +75,13 @@ class MVTModule(object):
     def check_indicators(self):
         """Check the results of this module against a provided list of
         indicators.
+
+
         """
         raise NotImplementedError
 
     def save_to_json(self):
-        """Save the collected results to a json file.
-        """
+        """Save the collected results to a json file."""
         if not self.output_folder:
             return
 
@@ -112,6 +111,7 @@ class MVTModule(object):
         """Serialize entry as JSON to deduplicate repeated entries
 
         :param timeline: List of entries from timeline to deduplicate
+
         """
         timeline_set = set()
         for record in timeline:
@@ -141,8 +141,7 @@ class MVTModule(object):
         self.timeline_detected = self._deduplicate_timeline(self.timeline_detected)
 
     def run(self):
-        """Run the main module procedure.
-        """
+        """Run the main module procedure."""
         raise NotImplementedError
 
 
@@ -190,6 +189,7 @@ def save_timeline(timeline, timeline_path):
 
     :param timeline: List of records to order and store
     :param timeline_path: Path to the csv file to store the timeline to
+
     """
     with io.open(timeline_path, "a+", encoding="utf-8") as handle:
         csvoutput = csv.writer(handle, delimiter=",", quotechar="\"")

mvt/common/url.py

@@ -268,6 +268,7 @@ class URL:
         :type url: str
         :returns: Domain name extracted from URL
         :rtype: str
+
         """
         # TODO: Properly handle exception.
         try:
@@ -282,6 +283,7 @@ class URL:
         :type url: str
         :returns: Top-level domain name extracted from URL
         :rtype: str
+
         """
         # TODO: Properly handle exception.
         try:
@@ -292,8 +294,11 @@ class URL:
     def check_if_shortened(self) -> bool:
         """Check if the URL is among list of shortener services.
 
+
         :returns: True if the URL is shortened, otherwise False
+
         :rtype: bool
+
         """
         if self.domain.lower() in SHORTENER_DOMAINS:
             self.is_shortened = True
@@ -301,8 +306,7 @@ class URL:
         return self.is_shortened
 
     def unshorten(self):
-        """Unshorten the URL by requesting an HTTP HEAD response.
-        """
+        """Unshorten the URL by requesting an HTTP HEAD response."""
         res = requests.head(self.url)
         if str(res.status_code).startswith("30"):
             return res.headers["Location"]

mvt/common/utils.py

@@ -16,6 +16,7 @@ def convert_mactime_to_unix(timestamp, from_2001=True):
     :param from_2001: bool: Whether to (Default value = True)
     :param from_2001: Default value = True)
     :returns: Unix epoch timestamp.
+
     """
     if not timestamp:
         return None
@@ -42,6 +43,7 @@ def convert_chrometime_to_unix(timestamp):
     :param timestamp: Chrome timestamp as int.
     :type timestamp: int
     :returns: Unix epoch timestamp.
+
     """
     epoch_start = datetime.datetime(1601, 1 , 1)
     delta = datetime.timedelta(microseconds=timestamp)
@@ -55,6 +57,7 @@ def convert_timestamp_to_iso(timestamp):
     :type timestamp: int
     :returns: ISO timestamp string in YYYY-mm-dd HH:MM:SS.ms format.
     :rtype: str
+
     """
     try:
         return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
@@ -67,6 +70,7 @@ def check_for_links(text):
     :param text: Any provided text.
     :type text: str
     :returns: Search results.
+
     """
     return re.findall("(?P<url>https?://[^\s]+)", text, re.IGNORECASE)
 
@@ -92,6 +96,7 @@ def keys_bytes_to_string(obj):
     :param obj: Object to convert from bytes to string.
     :returns: Object converted to string.
     :rtype: str
+
     """
     new_obj = {}
     if not isinstance(obj, dict):

mvt/common/version.py

@@ -6,7 +6,7 @@
 import requests
 from packaging import version
 
-MVT_VERSION = "1.2.8"
+MVT_VERSION = "1.2.11"
 
 def check_for_updates():
     res = requests.get("https://pypi.org/pypi/mvt/json")

mvt/ios/cli.py

@@ -38,6 +38,14 @@ def cli():
     logo()
 
 
+#==============================================================================
+# Command: version
+#==============================================================================
+@cli.command("version", help="Show the currently installed version of MVT")
+def version():
+    return
+
+
 #==============================================================================
 # Command: decrypt-backup
 #==============================================================================

mvt/ios/modules/fs/__init__.py

@@ -7,10 +7,12 @@ from .cache_files import CacheFiles
 from .filesystem import Filesystem
 from .net_netusage import Netusage
 from .safari_favicon import SafariFavicon
+from .shutdownlog import ShutdownLog
 from .version_history import IOSVersionHistory
 from .webkit_indexeddb import WebkitIndexedDB
 from .webkit_localstorage import WebkitLocalStorage
 from .webkit_safariviewservice import WebkitSafariViewService
 
-FS_MODULES = [CacheFiles, Filesystem, Netusage, SafariFavicon, IOSVersionHistory,
-              WebkitIndexedDB, WebkitLocalStorage, WebkitSafariViewService,]
+FS_MODULES = [CacheFiles, Filesystem, Netusage, SafariFavicon, ShutdownLog,
+              IOSVersionHistory, WebkitIndexedDB, WebkitLocalStorage,
+              WebkitSafariViewService,]

mvt/ios/modules/fs/shutdownlog.py

@@ -0,0 +1,81 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021 The MVT Project Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from mvt.common.utils import convert_mactime_to_unix, convert_timestamp_to_iso
+
+from ..base import IOSExtraction
+
+SHUTDOWN_LOG_PATH = [
+    "private/var/db/diagnostics/shutdown.log",
+]
+
+class ShutdownLog(IOSExtraction):
+    """This module extracts processes information from the shutdown log file."""
+
+    def __init__(self, file_path=None, base_folder=None, output_folder=None,
+                 fast_mode=False, log=None, results=[]):
+        super().__init__(file_path=file_path, base_folder=base_folder,
+                         output_folder=output_folder, fast_mode=fast_mode,
+                         log=log, results=results)
+
+    def serialize(self, record):
+        return {
+            "timestamp": record["isodate"],
+            "module": self.__class__.__name__,
+            "event": "shutdown",
+            "data": f"Client {record['client']} with PID {record['pid']} was running when the device was shut down",
+        }
+    
+    def check_indicators(self):
+        if not self.indicators:
+            return
+
+        for result in self.results:
+            for ioc in self.indicators.ioc_processes:
+                parts = result["client"].split("/")
+                if ioc in parts:
+                    self.log.warning("Found mention of a known malicious process \"%s\" in shutdown.log",
+                                     ioc)
+                    self.detected.append(result)
+
+    def process_shutdownlog(self, content):
+        current_processes = []
+        for line in content.split("\n"):
+            line = line.strip()
+
+            if line.startswith("remaining client pid:"):
+                current_processes.append({
+                    "pid": line[line.find("pid: ")+5:line.find(" (")],
+                    "client": line[line.find("(")+1:line.find(")")],
+                })
+            elif line.startswith("SIGTERM: "):
+                try:
+                    mac_timestamp = int(line[line.find("[")+1:line.find("]")])
+                except ValueError:
+                    try:
+                        start = line.find(" @")+2
+                        mac_timestamp = int(line[start:start+10])
+                    except:
+                        mac_timestamp = 0
+
+                timestamp = convert_mactime_to_unix(mac_timestamp, from_2001=False)
+                isodate = convert_timestamp_to_iso(timestamp)
+
+                for current_process in current_processes:
+                    self.results.append({
+                        "isodate": isodate,
+                        "pid": current_process["pid"],
+                        "client": current_process["client"],
+                    })
+
+                current_processes = []
+
+        self.results = sorted(self.results, key=lambda entry: entry["isodate"])
+
+    def run(self):
+        self._find_ios_database(root_paths=SHUTDOWN_LOG_PATH)
+        self.log.info("Found shutdown log at path: %s", self.file_path)
+        with open(self.file_path, "r") as handle:
+            self.process_shutdownlog(handle.read())

mvt/ios/versions.py

@@ -223,6 +223,7 @@ IPHONE_IOS_VERSIONS = [
     {"build": "18G69", "version": "14.7"},
     {"build": "18G82", "version": "14.7.1"},
     {"build": "18H17", "version": "14.8"},
+    {"build": "19A346", "version": "15.0"},
 ]
 
 def get_device_desc_from_id(identifier, devices_list=IPHONE_MODELS):

setup.py

@@ -26,7 +26,7 @@ requires = (
     # iOS dependencies:
     "iOSbackup>=0.9.912",
     # Android dependencies:
-    "adb-shell>=0.4.0",
+    "adb-shell>=0.4.1",
     "libusb1>=1.9.3",
 )