From 84d7716ef1a91bbd1fd1ca25e6670482c01f7fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Mon, 28 Oct 2024 11:46:05 +0100 Subject: [PATCH 1/3] Use local timestamp for Files module timeline. Most other Android timestamps appear to be local time. The results timeline is more useful if all the timestamps are consistent. I would prefer to use UTC, but that would mean converting all the other timestamps to UTC as well. We probably do not have sufficient information to do that accurately, especially if the device is moving between timezones.. --- src/mvt/android/artifacts/getprop.py | 11 ++++++++++ src/mvt/android/modules/androidqf/base.py | 25 ++++++++++++++++++++++ src/mvt/android/modules/androidqf/files.py | 18 ++++++++++++---- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/mvt/android/artifacts/getprop.py b/src/mvt/android/artifacts/getprop.py index a4d87e0..6c7030f 100644 --- a/src/mvt/android/artifacts/getprop.py +++ b/src/mvt/android/artifacts/getprop.py @@ -42,6 +42,17 @@ class GetProp(AndroidArtifact): entry = {"name": matches[0][0], "value": matches[0][1]} self.results.append(entry) + def get_device_timezone(self) -> str: + """ + Get the device timezone from the getprop results + + Used in other moduels to calculate the timezone offset + """ + for entry in self.results: + if entry["name"] == "persist.sys.timezone": + return entry["value"] + return None + def check_indicators(self) -> None: for entry in self.results: if entry["name"] in INTERESTING_PROPERTIES: diff --git a/src/mvt/android/modules/androidqf/base.py b/src/mvt/android/modules/androidqf/base.py index 4e2c68e..e28e3da 100644 --- a/src/mvt/android/modules/androidqf/base.py +++ b/src/mvt/android/modules/androidqf/base.py @@ -48,6 +48,31 @@ class AndroidQFModule(MVTModule): def _get_files_by_pattern(self, pattern: str): return fnmatch.filter(self.files, pattern) + def _get_device_timezone(self): + """ + Get the device timezone from the getprop.txt file. + + This is needed to map local timestamps stored in some + Android log files to UTC/timezone-aware timestamps. + """ + get_prop_files = self._get_files_by_pattern("*/getprop.txt") + prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8") + + from mvt.android.artifacts.getprop import GetProp + + properties_artifact = GetProp() + properties_artifact.parse(prop_data) + timezone = properties_artifact.get_device_timezone() + if timezone: + self.log.debug("Identified local phone timezone: %s", timezone) + return timezone + + self.log.warning( + "Could not find or determine local device timezone. " + "Some timestamps and timeline data may be incorrect." + ) + return None + def _get_file_content(self, file_path): if self.archive: handle = self.archive.open(file_path) diff --git a/src/mvt/android/modules/androidqf/files.py b/src/mvt/android/modules/androidqf/files.py index 18dee9d..baec7a5 100644 --- a/src/mvt/android/modules/androidqf/files.py +++ b/src/mvt/android/modules/androidqf/files.py @@ -6,6 +6,7 @@ import datetime import json import logging +from zoneinfo import ZoneInfo from typing import Optional, Union from mvt.android.modules.androidqf.base import AndroidQFModule @@ -106,6 +107,8 @@ class Files(AndroidQFModule): # TODO: adds SHA1 and MD5 when available in MVT def run(self) -> None: + device_timezone = ZoneInfo(self._get_device_timezone()) + for file in self._get_files_by_pattern("*/files.json"): rawdata = self._get_file_content(file).decode("utf-8", errors="ignore") try: @@ -120,11 +123,18 @@ class Files(AndroidQFModule): for file_data in data: for ts in ["access_time", "changed_time", "modified_time"]: if ts in file_data: - file_data[ts] = convert_datetime_to_iso( - datetime.datetime.fromtimestamp( - file_data[ts], tz=datetime.timezone.utc - ) + utc_timestamp = datetime.datetime.fromtimestamp( + file_data[ts], tz=datetime.timezone.utc ) + # Convert the UTC timestamp to local tiem on Android device's local timezone + local_timestamp = utc_timestamp.astimezone(device_timezone) + + # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we + # set the timestamp timezone to UTC, to avoid the timezone conversion again. + local_timestamp = local_timestamp.replace( + tzinfo=datetime.timezone.utc + ) + file_data[ts] = convert_datetime_to_iso(local_timestamp) self.results.append(file_data) From 39f78851ae5d646712b2a2ace21380c3f31ec363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Mon, 28 Oct 2024 11:49:30 +0100 Subject: [PATCH 2/3] Add file timestamp modules to add logs into timeline --- src/mvt/android/artifacts/file_timestamps.py | 43 ++++++++++++ .../modules/androidqf/logfile_timestamps.py | 65 +++++++++++++++++++ src/mvt/android/modules/bugreport/__init__.py | 2 + src/mvt/android/modules/bugreport/base.py | 1 + .../modules/bugreport/fs_timestamps.py | 65 +++++++++++++++++++ 5 files changed, 176 insertions(+) create mode 100644 src/mvt/android/artifacts/file_timestamps.py create mode 100644 src/mvt/android/modules/androidqf/logfile_timestamps.py create mode 100644 src/mvt/android/modules/bugreport/fs_timestamps.py diff --git a/src/mvt/android/artifacts/file_timestamps.py b/src/mvt/android/artifacts/file_timestamps.py new file mode 100644 index 0000000..aa2dc25 --- /dev/null +++ b/src/mvt/android/artifacts/file_timestamps.py @@ -0,0 +1,43 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +from typing import Union + +from .artifact import AndroidArtifact + + +class FileTimestampsArtifact(AndroidArtifact): + def serialize(self, record: dict) -> Union[dict, list]: + records = [] + + for ts in set( + [ + record.get("access_time"), + record.get("changed_time"), + record.get("modified_time"), + ] + ): + if not ts: + continue + + macb = "" + macb += "M" if ts == record.get("modified_time") else "-" + macb += "A" if ts == record.get("access_time") else "-" + macb += "C" if ts == record.get("changed_time") else "-" + macb += "-" + + msg = record["path"] + if record.get("context"): + msg += f" ({record['context']})" + + records.append( + { + "timestamp": ts, + "module": self.__class__.__name__, + "event": macb, + "data": msg, + } + ) + + return records diff --git a/src/mvt/android/modules/androidqf/logfile_timestamps.py b/src/mvt/android/modules/androidqf/logfile_timestamps.py new file mode 100644 index 0000000..b37851d --- /dev/null +++ b/src/mvt/android/modules/androidqf/logfile_timestamps.py @@ -0,0 +1,65 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import datetime +import logging +from typing import Optional + +from mvt.common.utils import convert_datetime_to_iso +from .base import AndroidQFModule +from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact + + +class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule): + """This module extracts records from battery daily updates.""" + + slug = "logfile_timestamps" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def _get_file_modification_time(self, file_path: str) -> dict: + if self.archive: + file_timetuple = self.archive.getinfo(file_path).date_time + return datetime.datetime(*file_timetuple) + else: + file_stat = os.stat(os.path.join(self.parent_path, file_path)) + return datetime.datetime.fromtimestamp(file_stat.st_mtime) + + def run(self) -> None: + filesystem_files = self._get_files_by_pattern("*/logs/*") + + self.results = [] + for file in filesystem_files: + # Only the modification time is available in the zip file metadata. + # The timezone is the local timezone of the machine the phone. + modification_time = self._get_file_modification_time(file) + self.results.append( + { + "path": file, + "modified_time": convert_datetime_to_iso(modification_time), + } + ) + + self.log.info( + "Extracted a total of %d filesystem timestamps from AndroidQF logs directory.", + len(self.results), + ) diff --git a/src/mvt/android/modules/bugreport/__init__.py b/src/mvt/android/modules/bugreport/__init__.py index 119c958..c5f715e 100644 --- a/src/mvt/android/modules/bugreport/__init__.py +++ b/src/mvt/android/modules/bugreport/__init__.py @@ -13,6 +13,7 @@ from .getprop import Getprop from .packages import Packages from .receivers import Receivers from .adb_state import DumpsysADBState +from .fs_timestamps import BugReportTimestamps BUGREPORT_MODULES = [ Accessibility, @@ -25,4 +26,5 @@ BUGREPORT_MODULES = [ Packages, Receivers, DumpsysADBState, + BugReportTimestamps, ] diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index d434116..f209815 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -6,6 +6,7 @@ import fnmatch import logging import os + from typing import List, Optional from zipfile import ZipFile diff --git a/src/mvt/android/modules/bugreport/fs_timestamps.py b/src/mvt/android/modules/bugreport/fs_timestamps.py new file mode 100644 index 0000000..a6d7783 --- /dev/null +++ b/src/mvt/android/modules/bugreport/fs_timestamps.py @@ -0,0 +1,65 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import logging +import datetime +from typing import Optional + +from mvt.common.utils import convert_datetime_to_iso +from .base import BugReportModule +from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact + + +class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): + """This module extracts records from battery daily updates.""" + + slug = "bugreport_timestamps" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def _get_file_modification_time(self, file_path: str) -> dict: + if self.zip_archive: + file_timetuple = self.zip_archive.getinfo(file_path).date_time + return datetime.datetime(*file_timetuple) + else: + file_stat = os.stat(os.path.join(self.extract_path, file_path)) + return datetime.datetime.fromtimestamp(file_stat.st_mtime) + + def run(self) -> None: + filesystem_files = self._get_files_by_pattern("FS/*") + + self.results = [] + for file in filesystem_files: + # Only the modification time is available in the zip file metadata. + # The timezone is the local timezone of the machine the phone. + modification_time = self._get_file_modification_time(file) + self.results.append( + { + "path": file, + "modified_time": convert_datetime_to_iso(modification_time), + } + ) + + self.log.info( + "Extracted a total of %d filesystem timestamps from bugreport.", + len(self.results), + ) From 4a14c97be3bbd469bd4aaad90f4cbd63f8e5638d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Mon, 28 Oct 2024 11:55:41 +0100 Subject: [PATCH 3/3] Handle case were we cannot load device timezone --- src/mvt/android/modules/androidqf/files.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mvt/android/modules/androidqf/files.py b/src/mvt/android/modules/androidqf/files.py index baec7a5..dbca3a7 100644 --- a/src/mvt/android/modules/androidqf/files.py +++ b/src/mvt/android/modules/androidqf/files.py @@ -107,7 +107,11 @@ class Files(AndroidQFModule): # TODO: adds SHA1 and MD5 when available in MVT def run(self) -> None: - device_timezone = ZoneInfo(self._get_device_timezone()) + if timezone := self._get_device_timezone(): + device_timezone = ZoneInfo(timezone) + else: + self.log.warning("Unable to determine device timezone, using UTC") + device_timezone = ZoneInfo("UTC") for file in self._get_files_by_pattern("*/files.json"): rawdata = self._get_file_content(file).decode("utf-8", errors="ignore")