Merge branch 'feature/android-sub-module-loading' into local-timeline-fixes

Most other Android timestamps appear to be local time. The
results timeline is more useful if all the timestamps
are consistent. I would prefer to use UTC, but that would
mean converting all the other timestamps to UTC as well. We probably
do not have sufficient information to do that accurately,
especially if the device is moving between timezones..

src/mvt/android/artifacts/file_timestamps.py

@@ -0,0 +1,43 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+from typing import Union
+
+from .artifact import AndroidArtifact
+
+
+class FileTimestampsArtifact(AndroidArtifact):
+    def serialize(self, record: dict) -> Union[dict, list]:
+        records = []
+
+        for ts in set(
+            [
+                record.get("access_time"),
+                record.get("changed_time"),
+                record.get("modified_time"),
+            ]
+        ):
+            if not ts:
+                continue
+
+            macb = ""
+            macb += "M" if ts == record.get("modified_time") else "-"
+            macb += "A" if ts == record.get("access_time") else "-"
+            macb += "C" if ts == record.get("changed_time") else "-"
+            macb += "-"
+
+            msg = record["path"]
+            if record.get("context"):
+                msg += f" ({record['context']})"
+
+            records.append(
+                {
+                    "timestamp": ts,
+                    "module": self.__class__.__name__,
+                    "event": macb,
+                    "data": msg,
+                }
+            )
+
+        return records

src/mvt/android/artifacts/getprop.py

@@ -42,6 +42,17 @@ class GetProp(AndroidArtifact):
             entry = {"name": matches[0][0], "value": matches[0][1]}
             self.results.append(entry)
 
+    def get_device_timezone(self) -> str:
+        """
+        Get the device timezone from the getprop results
+
+        Used in other moduels to calculate the timezone offset
+        """
+        for entry in self.results:
+            if entry["name"] == "persist.sys.timezone":
+                return entry["value"]
+        return None
+
     def check_indicators(self) -> None:
         for entry in self.results:
             if entry["name"] in INTERESTING_PROPERTIES:

src/mvt/android/cmd_check_androidqf.py

@@ -12,6 +12,8 @@ from typing import List, Optional
 from mvt.common.command import Command
 
 from .modules.androidqf import ANDROIDQF_MODULES
+from .modules.bugreport import BUGREPORT_MODULES
+from .modules.bugreport.base import BugReportModule
 
 log = logging.getLogger(__name__)
 
@@ -39,7 +41,11 @@ class CmdAndroidCheckAndroidQF(Command):
         )
 
         self.name = "check-androidqf"
-        self.modules = ANDROIDQF_MODULES
+
+        # We can load AndroidQF and bugreport modules here, as
+        # AndroidQF dump will contain a bugreport.
+        self.modules = ANDROIDQF_MODULES + BUGREPORT_MODULES
+        # TODO: Check how to namespace and deduplicate modules.
 
         self.format: Optional[str] = None
         self.archive: Optional[zipfile.ZipFile] = None
@@ -54,12 +60,44 @@ class CmdAndroidCheckAndroidQF(Command):
                 for fname in subfiles:
                     file_path = os.path.relpath(os.path.join(root, fname), parent_path)
                     self.files.append(file_path)
+
         elif os.path.isfile(self.target_path):
             self.format = "zip"
             self.archive = zipfile.ZipFile(self.target_path)
             self.files = self.archive.namelist()
 
+    def load_bugreport(self):
+        # Refactor this file list loading
+        # First we need to find the bugreport file location
+        bugreport_zip_path = None
+        for file_name in self.files:
+            if file_name.endswith("bugreport.zip"):
+                bugreport_zip_path = file_name
+                break
+        else:
+            self.log.warning("No bugreport.zip found in the AndroidQF dump")
+            return None
+
+        if self.format == "zip":
+            # Create handle to the bugreport.zip file inside the AndroidQF dump
+            handle = self.archive.open(bugreport_zip_path)
+            bugreport_zip = zipfile.ZipFile(handle)
+        else:
+            # Load the bugreport.zip file from the extracted AndroidQF dump on disk.
+            parent_path = Path(self.target_path).absolute().parent.as_posix()
+            bug_report_path = os.path.join(parent_path, bugreport_zip_path)
+            bugreport_zip = zipfile.ZipFile(bug_report_path)
+
+        return bugreport_zip
+
     def module_init(self, module):
+        if isinstance(module, BugReportModule):
+            bugreport_archive = self.load_bugreport()
+            if not bugreport_archive:
+                return
+            module.from_zip(bugreport_archive, bugreport_archive.namelist())
+            return
+
         if self.format == "zip":
             module.from_zip_file(self.archive, self.files)
         else:

src/mvt/android/modules/androidqf/base.py

@@ -48,6 +48,31 @@ class AndroidQFModule(MVTModule):
     def _get_files_by_pattern(self, pattern: str):
         return fnmatch.filter(self.files, pattern)
 
+    def _get_device_timezone(self):
+        """
+        Get the device timezone from the getprop.txt file.
+
+        This is needed to map local timestamps stored in some
+        Android log files to UTC/timezone-aware timestamps.
+        """
+        get_prop_files = self._get_files_by_pattern("*/getprop.txt")
+        prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8")
+
+        from mvt.android.artifacts.getprop import GetProp
+
+        properties_artifact = GetProp()
+        properties_artifact.parse(prop_data)
+        timezone = properties_artifact.get_device_timezone()
+        if timezone:
+            self.log.debug("Identified local phone timezone: %s", timezone)
+            return timezone
+
+        self.log.warning(
+            "Could not find or determine local device timezone. "
+            "Some timestamps and timeline data may be incorrect."
+        )
+        return None
+
     def _get_file_content(self, file_path):
         if self.archive:
             handle = self.archive.open(file_path)

src/mvt/android/modules/androidqf/files.py

@@ -6,6 +6,7 @@
 import datetime
 import json
 import logging
+from zoneinfo import ZoneInfo
 from typing import Optional, Union
 
 from mvt.android.modules.androidqf.base import AndroidQFModule
@@ -106,6 +107,12 @@ class Files(AndroidQFModule):
             # TODO: adds SHA1 and MD5 when available in MVT
 
     def run(self) -> None:
+        if timezone := self._get_device_timezone():
+            device_timezone = ZoneInfo(timezone)
+        else:
+            self.log.warning("Unable to determine device timezone, using UTC")
+            device_timezone = ZoneInfo("UTC")
+
         for file in self._get_files_by_pattern("*/files.json"):
             rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")
             try:
@@ -120,11 +127,18 @@ class Files(AndroidQFModule):
             for file_data in data:
                 for ts in ["access_time", "changed_time", "modified_time"]:
                     if ts in file_data:
-                        file_data[ts] = convert_datetime_to_iso(
+                        utc_timestamp = datetime.datetime.fromtimestamp(
-                            datetime.datetime.fromtimestamp(
+                            file_data[ts], tz=datetime.timezone.utc
-                                file_data[ts], tz=datetime.timezone.utc
-                            )
                         )
+                        # Convert the UTC timestamp to local tiem on Android device's local timezone
+                        local_timestamp = utc_timestamp.astimezone(device_timezone)
+
+                        # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we
+                        # set the timestamp timezone to UTC, to avoid the timezone conversion again.
+                        local_timestamp = local_timestamp.replace(
+                            tzinfo=datetime.timezone.utc
+                        )
+                        file_data[ts] = convert_datetime_to_iso(local_timestamp)
 
                 self.results.append(file_data)
 

src/mvt/android/modules/androidqf/logfile_timestamps.py

@@ -0,0 +1,65 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+import datetime
+import logging
+from typing import Optional
+
+from mvt.common.utils import convert_datetime_to_iso
+from .base import AndroidQFModule
+from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
+
+
+class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "logfile_timestamps"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def _get_file_modification_time(self, file_path: str) -> dict:
+        if self.archive:
+            file_timetuple = self.archive.getinfo(file_path).date_time
+            return datetime.datetime(*file_timetuple)
+        else:
+            file_stat = os.stat(os.path.join(self.parent_path, file_path))
+            return datetime.datetime.fromtimestamp(file_stat.st_mtime)
+
+    def run(self) -> None:
+        filesystem_files = self._get_files_by_pattern("*/logs/*")
+
+        self.results = []
+        for file in filesystem_files:
+            # Only the modification time is available in the zip file metadata.
+            # The timezone is the local timezone of the machine the phone.
+            modification_time = self._get_file_modification_time(file)
+            self.results.append(
+                {
+                    "path": file,
+                    "modified_time": convert_datetime_to_iso(modification_time),
+                }
+            )
+
+        self.log.info(
+            "Extracted a total of %d filesystem timestamps from AndroidQF logs directory.",
+            len(self.results),
+        )

src/mvt/android/modules/bugreport/__init__.py

@@ -13,6 +13,7 @@ from .getprop import Getprop
 from .packages import Packages
 from .receivers import Receivers
 from .adb_state import DumpsysADBState
+from .fs_timestamps import BugReportTimestamps
 
 BUGREPORT_MODULES = [
     Accessibility,
@@ -25,4 +26,5 @@ BUGREPORT_MODULES = [
     Packages,
     Receivers,
     DumpsysADBState,
+    BugReportTimestamps,
 ]

src/mvt/android/modules/bugreport/base.py

@@ -6,6 +6,7 @@
 import fnmatch
 import logging
 import os
+
 from typing import List, Optional
 from zipfile import ZipFile
 

src/mvt/android/modules/bugreport/fs_timestamps.py

@@ -0,0 +1,65 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+import os
+import logging
+import datetime
+from typing import Optional
+
+from mvt.common.utils import convert_datetime_to_iso
+from .base import BugReportModule
+from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
+
+
+class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
+    """This module extracts records from battery daily updates."""
+
+    slug = "bugreport_timestamps"
+
+    def __init__(
+        self,
+        file_path: Optional[str] = None,
+        target_path: Optional[str] = None,
+        results_path: Optional[str] = None,
+        module_options: Optional[dict] = None,
+        log: logging.Logger = logging.getLogger(__name__),
+        results: Optional[list] = None,
+    ) -> None:
+        super().__init__(
+            file_path=file_path,
+            target_path=target_path,
+            results_path=results_path,
+            module_options=module_options,
+            log=log,
+            results=results,
+        )
+
+    def _get_file_modification_time(self, file_path: str) -> dict:
+        if self.zip_archive:
+            file_timetuple = self.zip_archive.getinfo(file_path).date_time
+            return datetime.datetime(*file_timetuple)
+        else:
+            file_stat = os.stat(os.path.join(self.extract_path, file_path))
+            return datetime.datetime.fromtimestamp(file_stat.st_mtime)
+
+    def run(self) -> None:
+        filesystem_files = self._get_files_by_pattern("FS/*")
+
+        self.results = []
+        for file in filesystem_files:
+            # Only the modification time is available in the zip file metadata.
+            # The timezone is the local timezone of the machine the phone.
+            modification_time = self._get_file_modification_time(file)
+            self.results.append(
+                {
+                    "path": file,
+                    "modified_time": convert_datetime_to_iso(modification_time),
+                }
+            )
+
+        self.log.info(
+            "Extracted a total of %d filesystem timestamps from bugreport.",
+            len(self.results),
+        )