Extract all messages from SMS and WhatsApp (#337)

mvt/android/modules/adb/sms.py

@@ -43,7 +43,7 @@ FROM sms;
 
 
 class SMS(AndroidExtraction):
-    """This module extracts all SMS messages containing links."""
+    """This module extracts all SMS messages."""
 
     def __init__(
         self,
@@ -77,8 +77,10 @@ class SMS(AndroidExtraction):
             if "body" not in message:
                 continue
 
-            # TODO: check links exported from the body previously.
-            message_links = check_for_links(message["body"])
+            message_links = message.get("links", [])
+            if message_links == []:
+                message_links = check_for_links(message["body"])
+
             if self.indicators.check_domains(message_links):
                 self.detected.append(message)
 
@@ -106,15 +108,16 @@ class SMS(AndroidExtraction):
             message["direction"] = ("received" if message["incoming"] == 1 else "sent")
             message["isodate"] = convert_unix_to_iso(message["timestamp"])
 
-            # If we find links in the messages or if they are empty we add
-            # them to the list of results.
-            if check_for_links(message["body"]) or message["body"].strip() == "":
-                self.results.append(message)
+            # Extract links in the message body
+            links = check_for_links(message["body"])
+            message["links"] = links
+
+            self.results.append(message)
 
         cur.close()
         conn.close()
 
-        self.log.info("Extracted a total of %d SMS messages containing links",
+        self.log.info("Extracted a total of %d SMS messages",
                       len(self.results))
 
     def _extract_sms_adb(self) -> None:
@@ -137,7 +140,7 @@ class SMS(AndroidExtraction):
                           "Android Backup Extractor")
             return
 
-        self.log.info("Extracted a total of %d SMS messages containing links",
+        self.log.info("Extracted a total of %d SMS messages",
                       len(self.results))
 
     def run(self) -> None:

mvt/android/modules/backup/sms.py

@@ -8,6 +8,7 @@ from typing import Optional
 
 from mvt.android.modules.backup.base import BackupExtraction
 from mvt.android.parsers.backup import parse_sms_file
+from mvt.common.utils import check_for_links
 
 
 class SMS(BackupExtraction):
@@ -34,7 +35,11 @@ class SMS(BackupExtraction):
             if "body" not in message:
                 continue
 
-            if self.indicators.check_domains(message["links"]):
+            message_links = message.get("links", [])
+            if message_links == []:
+                message_links = check_for_links(message.get("text", ""))
+
+            if self.indicators.check_domains(message_links):
                 self.detected.append(message)
 
     def run(self) -> None:
@@ -50,5 +55,5 @@ class SMS(BackupExtraction):
             data = self._get_file_content(file)
             self.results.extend(parse_sms_file(data))
 
-        self.log.info("Extracted a total of %d SMS & MMS messages containing links",
+        self.log.info("Extracted a total of %d SMS & MMS messages",
                       len(self.results))

mvt/android/parsers/backup.py

@@ -218,10 +218,9 @@ def parse_sms_file(data):
         entry["isodate"] = convert_unix_to_iso(int(entry["date"]) / 1000)
         entry["direction"] = ("sent" if int(entry["date_sent"]) else "received")
 
-        # If we find links in the messages or if they are empty we add them to
-        # the list.
+        # Extract links from the body
         if message_links or entry["body"].strip() == "":
             entry["links"] = message_links
-            res.append(entry)
+        res.append(entry)
 
     return res

mvt/ios/modules/mixed/sms.py

@@ -51,7 +51,10 @@ class SMS(IOSExtraction):
             return
 
         for result in self.results:
-            message_links = check_for_links(result.get("text", ""))
+            message_links = result.get("links", [])
+            # Making sure not link was ignored
+            if message_links == []:
+                message_links = check_for_links(result.get("text", ""))
             ioc = self.indicators.check_domains(message_links)
             if ioc:
                 result["matched_indicator"] = ioc
@@ -118,18 +121,15 @@ class SMS(IOSExtraction):
             if message.get("text", "").startswith(alert):
                 self.log.warning("Apple warning about state-sponsored attack received on the %s",
                                  message["isodate"])
-                self.results.append(message)
             else:
                 # Extract links from the SMS message.
                 message_links = check_for_links(message.get("text", ""))
+                message["links"] = message_links
 
-                # If we find links in the messages or if they are empty we add
-                # them to the list.
-                if message_links or message.get("text", "").strip() == "":
-                    self.results.append(message)
+            self.results.append(message)
 
         cur.close()
         conn.close()
 
-        self.log.info("Extracted a total of %d SMS messages containing links",
+        self.log.info("Extracted a total of %d SMS messages",
                       len(self.results))

mvt/ios/modules/mixed/whatsapp.py

@@ -112,14 +112,13 @@ class Whatsapp(IOSExtraction):
                         or link.startswith("https://mmg.whatsapp.net/")):
                     filtered_links.append(link)
 
-            # If we find messages with links, or if there's an empty message
-            # we add it to the results list.
+            # Add all the links found to the record
             if filtered_links or (message.get("ZTEXT") or "").strip() == "":
                 message["links"] = list(set(filtered_links))
-                self.results.append(message)
+            self.results.append(message)
 
         cur.close()
         conn.close()
 
-        self.log.info("Extracted a total of %d WhatsApp messages containing links",
+        self.log.info("Extracted a total of %d WhatsApp messages",
                       len(self.results))