Merge branch 'mvt-project:main' into extract-key

mvt/android/modules/adb/base.py

@@ -4,6 +4,8 @@
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
 import os
+import random
+import string
 import sys
 import time
 import logging
@@ -96,7 +98,7 @@ class AndroidExtraction(MVTModule):
         """Check if we have a `su` binary on the Android device.
         :returns: Boolean indicating whether a `su` binary is present or not
         """
-        return bool(self._adb_command("[ ! -f /sbin/su ] || echo 1"))
+        return bool(self._adb_command("command -v su"))
 
     def _adb_root_or_die(self):
         """Check if we have a `su` binary, otherwise raise an Exception.
@@ -111,7 +113,7 @@ class AndroidExtraction(MVTModule):
         """
         return self._adb_command(f"su -c {command}")
 
-    def _adb_download(self, remote_path, local_path, progress_callback=None):
+    def _adb_download(self, remote_path, local_path, progress_callback=None, retry_root=True):
         """Download a file form the device.
         :param remote_path: Path to download from the device
         :param local_path: Path to where to locally store the copy of the file
@@ -119,6 +121,37 @@ class AndroidExtraction(MVTModule):
         """
         try:
             self.device.pull(remote_path, local_path, progress_callback)
+        except AdbCommandFailureException as e:
+            if retry_root:
+                self._adb_download_root(remote_path, local_path, progress_callback)
+            else:
+                raise Exception(f"Unable to download file {remote_path}: {e}")
+    
+    def _adb_download_root(self, remote_path, local_path, progress_callback=None):
+        try:
+            # Check if we have root, if not raise an Exception.
+            self._adb_root_or_die()
+
+            # We generate a random temporary filename.
+            tmp_filename = "tmp_" + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=10))
+
+            # We create a temporary local file.
+            new_remote_path = f"/sdcard/{tmp_filename}"
+
+            # We copy the file from the data folder to /sdcard/.
+            cp = self._adb_command_as_root(f"cp {remote_path} {new_remote_path}")
+            if cp.startswith("cp: ") and "No such file or directory" in cp:
+                raise Exception(f"Unable to process file {remote_path}: File not found")
+            elif cp.startswith("cp: ") and "Permission denied" in cp:
+                raise Exception(f"Unable to process file {remote_path}: Permission denied")
+
+            # We download from /sdcard/ to the local temporary file.
+            # If it doesn't work now, don't try again (retry_root=False)
+            self._adb_download(new_remote_path, local_path, retry_root=False)
+
+            # Delete the copy on /sdcard/.
+            self._adb_command(f"rm -rf {new_remote_path}")
+            
         except AdbCommandFailureException as e:
             raise Exception(f"Unable to download file {remote_path}: {e}")
 

mvt/android/modules/adb/whatsapp.py

@@ -6,6 +6,7 @@
 import os
 import sqlite3
 import logging
+import base64
 
 from .base import AndroidExtraction
 from mvt.common.utils import convert_timestamp_to_iso, check_for_links
@@ -69,6 +70,8 @@ class Whatsapp(AndroidExtraction):
 
             # If we find links in the messages or if they are empty we add them to the list.
             if check_for_links(message["data"]) or message["data"].strip() == "":
+                if (message.get('thumb_image') is not None):
+                    message['thumb_image'] = base64.b64encode(message['thumb_image'])
                 messages.append(message)
 
         cur.close()

mvt/common/module.py

@@ -13,6 +13,12 @@ import simplejson as json
 
 from .indicators import Indicators
 
+class DatabaseNotFoundError(Exception):
+    pass
+
+class DatabaseCorruptedError(Exception):
+    pass
+
 class MVTModule(object):
     """This class provides a base for all extraction modules."""
 
@@ -83,7 +89,11 @@ class MVTModule(object):
             results_file_name = f"{name}.json"
             results_json_path = os.path.join(self.output_folder, results_file_name)
             with open(results_json_path, "w") as handle:
-                json.dump(self.results, handle, indent=4)
+                try:
+                    json.dump(self.results, handle, indent=4)
+                except Exception as e:
+                    self.log.error("Unable to store results of module %s to file %s: %s",
+                                   self.__class__.__name__, results_file_name, e)
 
         if self.detected:
             detected_file_name = f"{name}_detected.json"
@@ -99,17 +109,19 @@ class MVTModule(object):
         """
         for result in self.results:
             record = self.serialize(result)
-            if type(record) == list:
-                self.timeline.extend(record)
-            else:
-                self.timeline.append(record)
+            if record:
+                if type(record) == list:
+                    self.timeline.extend(record)
+                else:
+                    self.timeline.append(record)
 
         for detected in self.detected:
             record = self.serialize(detected)
-            if type(record) == list:
-                self.timeline_detected.extend(record)
-            else:
-                self.timeline_detected.append(record)
+            if record:
+                if type(record) == list:
+                    self.timeline_detected.extend(record)
+                else:
+                    self.timeline_detected.append(record)
 
         # De-duplicate timeline entries
         self.timeline = self.timeline_deduplicate(self.timeline)
@@ -136,8 +148,11 @@ def run_module(module):
     except NotImplementedError:
         module.log.exception("The run() procedure of module %s was not implemented yet!",
                              module.__class__.__name__)
-    except FileNotFoundError as e:
-        module.log.error("There might be no data to extract by module %s: %s",
+    except DatabaseNotFoundError as e:
+        module.log.info("There might be no data to extract by module %s: %s",
+                        module.__class__.__name__, e)
+    except DatabaseCorruptedError as e:
+        module.log.error("The %s module database seems to be corrupted and recovery failed: %s",
                          module.__class__.__name__, e)
     except Exception as e:
         module.log.exception("Error in running extraction from module %s: %s",

mvt/ios/modules/fs/base.py

@@ -3,10 +3,15 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
+import io
 import os
 import glob
+import shutil
+import sqlite3
+import subprocess
 
 from mvt.common.module import MVTModule
+from mvt.common.module import DatabaseNotFoundError, DatabaseCorruptedError
 
 class IOSExtraction(MVTModule):
     """This class provides a base for all iOS filesystem/backup extraction modules."""
@@ -15,6 +20,31 @@ class IOSExtraction(MVTModule):
     is_fs_dump = False
     is_sysdiagnose = False
 
+    def _recover_database(self, file_path):
+        """Tries to recover a malformed database by running a .clone command.
+        :param file_path: Path to the malformed database file.
+        """
+        # TODO: Find a better solution.
+
+        self.log.info("Database at path %s is malformed. Trying to recover...", file_path)
+
+        if not os.path.exists(file_path):
+            return
+
+        if not shutil.which("sqlite3"):
+            raise DatabaseCorruptedError("Unable to recover without sqlite3 binary. Please install sqlite3!")
+
+        bak_path = f"{file_path}.bak"
+        shutil.move(file_path, bak_path)
+
+        cmd = f"sqlite3 {bak_path} '.clone {file_path}'"
+        ret = subprocess.call(cmd, shell=True, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        if ret != 0:
+            raise DatabaseCorruptedError("Recovery of database failed")
+
+        self.log.info("Database at path %s recovered successfully!", file_path)
+
     def _find_ios_database(self, backup_ids=None, root_paths=[]):
         """Try to locate the module's database file from either an iTunes
         backup or a full filesystem dump.
@@ -52,4 +82,20 @@ class IOSExtraction(MVTModule):
         if file_path:
             self.file_path = file_path
         else:
-            raise FileNotFoundError("Unable to find the module's database file")
+            raise DatabaseNotFoundError("Unable to find the module's database file")
+
+        # Check if the database is corrupted.
+        conn = sqlite3.connect(self.file_path)
+        cur = conn.cursor()
+
+        try:
+            recover = False
+            cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        except sqlite3.DatabaseError as e:
+            if "database disk image is malformed" in str(e):
+                recover = True
+        finally:
+            conn.close()
+
+        if recover:
+            self._recover_database(self.file_path)

mvt/ios/modules/fs/locationd.py

@@ -62,7 +62,7 @@ class LocationdClients(IOSExtraction):
                 result["package"] = app
                 for ts in self.timestamps:
                     if ts in result.keys():
-                        result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[date]))
+                        result[ts] = convert_timestamp_to_iso(convert_mactime_to_unix(result[ts]))
 
                 self.results.append(result)
 

mvt/ios/modules/fs/manifest.py

@@ -40,6 +40,8 @@ class Manifest(IOSExtraction):
 
     def serialize(self, record):
         records = []
+        if "modified" not in record or "statusChanged" not in record:
+            return
         for ts in set([record["created"], record["modified"], record["statusChanged"]]):
             macb = ""
             macb += "M" if ts == record["modified"] else "-"
@@ -63,12 +65,15 @@ class Manifest(IOSExtraction):
         for result in self.results:
             if not "relativePath" in result:
                 continue
-
-            if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain":
-                self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain")
-                self.detected.append(result)
+            if not result["relativePath"]:
                 continue
 
+            if result["domain"]:
+                if os.path.basename(result["relativePath"]) == "com.apple.CrashReporter.plist" and result["domain"] == "RootDomain":
+                    self.log.warning("Found a potentially suspicious \"com.apple.CrashReporter.plist\" file created in RootDomain")
+                    self.detected.append(result)
+                    continue
+
             if self.indicators.check_file(result["relativePath"]):
                 self.log.warning("Found a known malicious file at path: %s", result["relativePath"])
                 self.detected.append(result)

mvt/ios/modules/fs/net_base.py

@@ -167,6 +167,10 @@ class NetBase(IOSExtraction):
         results_by_proc = {proc["proc_id"]: proc for proc in self.results if proc["proc_id"]}
         all_proc_id = sorted(results_by_proc.keys())
 
+        # Fix issue #108
+        if not all_proc_id:
+            return
+
         missing_procs, last_proc_id = {}, None
         for proc_id in range(min(all_proc_id), max(all_proc_id)):
             if proc_id not in all_proc_id:

mvt/ios/versions.py

@@ -102,6 +102,7 @@ IPHONE_IOS_VERSIONS = [
     {"build": "10A405", "version": "6.0"},
     {"build": "11B601", "version": "7.0.5"},
     {"build": "18F72", "version": "14.6"},
+    {"build": "18G69", "version": "14.7"},
     {"build": "18E199", "version": "14.5"},
     {"build": "18E212", "version": "14.5.1"},
     {"build": "18D52", "version": "14.4"},