From 4e97e85350be3b289a635bf6a6e72b1a13262860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Thu, 6 Feb 2025 20:51:15 +0100 Subject: [PATCH] Load Android device timezone info and add additional file modification logs (#567) * Use local timestamp for Files module timeline. Most other Android timestamps appear to be local time. The results timeline is more useful if all the timestamps are consistent. I would prefer to use UTC, but that would mean converting all the other timestamps to UTC as well. We probably do not have sufficient information to do that accurately, especially if the device is moving between timezones.. * Add file timestamp modules to add logs into timeline * Handle case were we cannot load device timezone * Fix crash if prop file does not exist * Move _get_file_modification_time to BugReportModule * Add backport for timezone and fix Tombstone module to use local time. * Fix import for backported Zoneinfo * Fix ruff error --- pyproject.toml | 1 + src/mvt/android/artifacts/file_timestamps.py | 43 ++++++++++++ src/mvt/android/artifacts/getprop.py | 11 ++++ .../android/artifacts/tombstone_crashes.py | 9 ++- src/mvt/android/modules/androidqf/base.py | 31 +++++++++ src/mvt/android/modules/androidqf/files.py | 26 ++++++-- .../modules/androidqf/logfile_timestamps.py | 65 +++++++++++++++++++ src/mvt/android/modules/bugreport/__init__.py | 2 + src/mvt/android/modules/bugreport/base.py | 1 + .../modules/bugreport/fs_timestamps.py | 55 ++++++++++++++++ src/mvt/common/command.py | 12 +++- src/mvt/common/module.py | 9 ++- tests/android/test_artifact_tombstones.py | 8 ++- 13 files changed, 260 insertions(+), 13 deletions(-) create mode 100644 src/mvt/android/artifacts/file_timestamps.py create mode 100644 src/mvt/android/modules/androidqf/logfile_timestamps.py create mode 100644 src/mvt/android/modules/bugreport/fs_timestamps.py diff --git a/pyproject.toml b/pyproject.toml index 96971b7..eec0824 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "betterproto >=1.2.0", "pydantic >= 2.10.0", "pydantic-settings >= 2.7.0", + 'backports.zoneinfo; python_version < "3.9"', ] requires-python = ">= 3.8" diff --git a/src/mvt/android/artifacts/file_timestamps.py b/src/mvt/android/artifacts/file_timestamps.py new file mode 100644 index 0000000..aa2dc25 --- /dev/null +++ b/src/mvt/android/artifacts/file_timestamps.py @@ -0,0 +1,43 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +from typing import Union + +from .artifact import AndroidArtifact + + +class FileTimestampsArtifact(AndroidArtifact): + def serialize(self, record: dict) -> Union[dict, list]: + records = [] + + for ts in set( + [ + record.get("access_time"), + record.get("changed_time"), + record.get("modified_time"), + ] + ): + if not ts: + continue + + macb = "" + macb += "M" if ts == record.get("modified_time") else "-" + macb += "A" if ts == record.get("access_time") else "-" + macb += "C" if ts == record.get("changed_time") else "-" + macb += "-" + + msg = record["path"] + if record.get("context"): + msg += f" ({record['context']})" + + records.append( + { + "timestamp": ts, + "module": self.__class__.__name__, + "event": macb, + "data": msg, + } + ) + + return records diff --git a/src/mvt/android/artifacts/getprop.py b/src/mvt/android/artifacts/getprop.py index a4d87e0..6c7030f 100644 --- a/src/mvt/android/artifacts/getprop.py +++ b/src/mvt/android/artifacts/getprop.py @@ -42,6 +42,17 @@ class GetProp(AndroidArtifact): entry = {"name": matches[0][0], "value": matches[0][1]} self.results.append(entry) + def get_device_timezone(self) -> str: + """ + Get the device timezone from the getprop results + + Used in other moduels to calculate the timezone offset + """ + for entry in self.results: + if entry["name"] == "persist.sys.timezone": + return entry["value"] + return None + def check_indicators(self) -> None: for entry in self.results: if entry["name"] in INTERESTING_PROPERTIES: diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index 62c329a..3827154 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -81,8 +81,8 @@ class TombstoneCrashArtifact(AndroidArtifact): "module": self.__class__.__name__, "event": "Tombstone", "data": ( - f"Crash in '{record['process_name']}' process running as UID '{record['uid']}' at " - f"{record['timestamp']}. Crash type '{record['signal_info']['name']}' with code '{record['signal_info']['code_name']}'" + f"Crash in '{record['process_name']}' process running as UID '{record['uid']}' in file '{record['file_name']}' " + f"Crash type '{record['signal_info']['name']}' with code '{record['signal_info']['code_name']}'" ), } @@ -258,7 +258,10 @@ class TombstoneCrashArtifact(AndroidArtifact): timestamp_parsed = datetime.datetime.strptime( timestamp_without_micro, "%Y-%m-%d %H:%M:%S%z" ) - return convert_datetime_to_iso(timestamp_parsed) + + # HACK: Swap the local timestamp to UTC, so keep the original time and avoid timezone conversion. + local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc) + return convert_datetime_to_iso(local_timestamp) @staticmethod def _proccess_name_from_thread(tombstone_dict: dict) -> str: diff --git a/src/mvt/android/modules/androidqf/base.py b/src/mvt/android/modules/androidqf/base.py index 4e2c68e..d871059 100644 --- a/src/mvt/android/modules/androidqf/base.py +++ b/src/mvt/android/modules/androidqf/base.py @@ -48,6 +48,37 @@ class AndroidQFModule(MVTModule): def _get_files_by_pattern(self, pattern: str): return fnmatch.filter(self.files, pattern) + def _get_device_timezone(self): + """ + Get the device timezone from the getprop.txt file. + + This is needed to map local timestamps stored in some + Android log files to UTC/timezone-aware timestamps. + """ + get_prop_files = self._get_files_by_pattern("*/getprop.txt") + if not get_prop_files: + self.log.warning( + "Could not find getprop.txt file. " + "Some timestamps and timeline data may be incorrect." + ) + return None + + from mvt.android.artifacts.getprop import GetProp + + properties_artifact = GetProp() + prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8") + properties_artifact.parse(prop_data) + timezone = properties_artifact.get_device_timezone() + if timezone: + self.log.debug("Identified local phone timezone: %s", timezone) + return timezone + + self.log.warning( + "Could not find or determine local device timezone. " + "Some timestamps and timeline data may be incorrect." + ) + return None + def _get_file_content(self, file_path): if self.archive: handle = self.archive.open(file_path) diff --git a/src/mvt/android/modules/androidqf/files.py b/src/mvt/android/modules/androidqf/files.py index fde7ca1..22b832c 100644 --- a/src/mvt/android/modules/androidqf/files.py +++ b/src/mvt/android/modules/androidqf/files.py @@ -6,6 +6,11 @@ import datetime import json import logging + +try: + import zoneinfo +except ImportError: + from backports import zoneinfo from typing import Optional, Union from mvt.android.modules.androidqf.base import AndroidQFModule @@ -106,6 +111,12 @@ class Files(AndroidQFModule): # TODO: adds SHA1 and MD5 when available in MVT def run(self) -> None: + if timezone := self._get_device_timezone(): + device_timezone = zoneinfo.ZoneInfo(timezone) + else: + self.log.warning("Unable to determine device timezone, using UTC") + device_timezone = zoneinfo.ZoneInfo("UTC") + for file in self._get_files_by_pattern("*/files.json"): rawdata = self._get_file_content(file).decode("utf-8", errors="ignore") try: @@ -120,11 +131,18 @@ class Files(AndroidQFModule): for file_data in data: for ts in ["access_time", "changed_time", "modified_time"]: if ts in file_data: - file_data[ts] = convert_datetime_to_iso( - datetime.datetime.fromtimestamp( - file_data[ts], tz=datetime.timezone.utc - ) + utc_timestamp = datetime.datetime.fromtimestamp( + file_data[ts], tz=datetime.timezone.utc ) + # Convert the UTC timestamp to local tiem on Android device's local timezone + local_timestamp = utc_timestamp.astimezone(device_timezone) + + # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we + # set the timestamp timezone to UTC, to avoid the timezone conversion again. + local_timestamp = local_timestamp.replace( + tzinfo=datetime.timezone.utc + ) + file_data[ts] = convert_datetime_to_iso(local_timestamp) self.results.append(file_data) diff --git a/src/mvt/android/modules/androidqf/logfile_timestamps.py b/src/mvt/android/modules/androidqf/logfile_timestamps.py new file mode 100644 index 0000000..b37851d --- /dev/null +++ b/src/mvt/android/modules/androidqf/logfile_timestamps.py @@ -0,0 +1,65 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import datetime +import logging +from typing import Optional + +from mvt.common.utils import convert_datetime_to_iso +from .base import AndroidQFModule +from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact + + +class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule): + """This module extracts records from battery daily updates.""" + + slug = "logfile_timestamps" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def _get_file_modification_time(self, file_path: str) -> dict: + if self.archive: + file_timetuple = self.archive.getinfo(file_path).date_time + return datetime.datetime(*file_timetuple) + else: + file_stat = os.stat(os.path.join(self.parent_path, file_path)) + return datetime.datetime.fromtimestamp(file_stat.st_mtime) + + def run(self) -> None: + filesystem_files = self._get_files_by_pattern("*/logs/*") + + self.results = [] + for file in filesystem_files: + # Only the modification time is available in the zip file metadata. + # The timezone is the local timezone of the machine the phone. + modification_time = self._get_file_modification_time(file) + self.results.append( + { + "path": file, + "modified_time": convert_datetime_to_iso(modification_time), + } + ) + + self.log.info( + "Extracted a total of %d filesystem timestamps from AndroidQF logs directory.", + len(self.results), + ) diff --git a/src/mvt/android/modules/bugreport/__init__.py b/src/mvt/android/modules/bugreport/__init__.py index 6945629..b5a1247 100644 --- a/src/mvt/android/modules/bugreport/__init__.py +++ b/src/mvt/android/modules/bugreport/__init__.py @@ -14,6 +14,7 @@ from .packages import Packages from .platform_compat import PlatformCompat from .receivers import Receivers from .adb_state import DumpsysADBState +from .fs_timestamps import BugReportTimestamps from .tombstones import Tombstones BUGREPORT_MODULES = [ @@ -28,5 +29,6 @@ BUGREPORT_MODULES = [ PlatformCompat, Receivers, DumpsysADBState, + BugReportTimestamps, Tombstones, ] diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index bf98dca..77802b2 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -6,6 +6,7 @@ import datetime import fnmatch import logging import os + from typing import List, Optional from zipfile import ZipFile diff --git a/src/mvt/android/modules/bugreport/fs_timestamps.py b/src/mvt/android/modules/bugreport/fs_timestamps.py new file mode 100644 index 0000000..14e1cd1 --- /dev/null +++ b/src/mvt/android/modules/bugreport/fs_timestamps.py @@ -0,0 +1,55 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +from typing import Optional + +from mvt.common.utils import convert_datetime_to_iso +from .base import BugReportModule +from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact + + +class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): + """This module extracts records from battery daily updates.""" + + slug = "bugreport_timestamps" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def run(self) -> None: + filesystem_files = self._get_files_by_pattern("FS/*") + + self.results = [] + for file in filesystem_files: + # Only the modification time is available in the zip file metadata. + # The timezone is the local timezone of the machine the phone. + modification_time = self._get_file_modification_time(file) + self.results.append( + { + "path": file, + "modified_time": convert_datetime_to_iso(modification_time), + } + ) + + self.log.info( + "Extracted a total of %d filesystem timestamps from bugreport.", + len(self.results), + ) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index 01309b8..7f65843 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -101,15 +101,25 @@ class Command: if not self.results_path: return + # We use local timestamps in the timeline on Android as many + # logs do not contain timezone information. + if type(self).__name__.startswith("CmdAndroid"): + is_utc = False + else: + is_utc = True + if len(self.timeline) > 0: save_timeline( - self.timeline, os.path.join(self.results_path, "timeline.csv") + self.timeline, + os.path.join(self.results_path, "timeline.csv"), + is_utc=is_utc, ) if len(self.timeline_detected) > 0: save_timeline( self.timeline_detected, os.path.join(self.results_path, "timeline_detected.csv"), + is_utc=is_utc, ) def _store_info(self) -> None: diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index 03e504f..1468cf4 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -227,7 +227,7 @@ def run_module(module: MVTModule) -> None: module.save_to_json() -def save_timeline(timeline: list, timeline_path: str) -> None: +def save_timeline(timeline: list, timeline_path: str, is_utc: bool = True) -> None: """Save the timeline in a csv file. :param timeline: List of records to order and store @@ -238,7 +238,12 @@ def save_timeline(timeline: list, timeline_path: str) -> None: csvoutput = csv.writer( handle, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, escapechar="\\" ) - csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"]) + + if is_utc: + timestamp_header = "UTC Timestamp" + else: + timestamp_header = "Device Local Timestamp" + csvoutput.writerow([timestamp_header, "Plugin", "Event", "Description"]) for event in sorted( timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else "" diff --git a/tests/android/test_artifact_tombstones.py b/tests/android/test_artifact_tombstones.py index b837d73..487eb21 100644 --- a/tests/android/test_artifact_tombstones.py +++ b/tests/android/test_artifact_tombstones.py @@ -60,6 +60,8 @@ class TestTombstoneCrashArtifact: assert tombstone_result.get("pid") == 25541 assert tombstone_result.get("process_name") == "mtk.ape.decoder" - # Check if the timestamp is correctly parsed, and converted to UTC - # Original is in +0200: 2023-04-12 12:32:40.518290770+0200, result should be 2023-04-12 10:32:40.000000+0000 - assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.000000" + # With Android logs we want to keep timestamps as device local time for consistency. + # We often don't know the time offset for a log entry and so can't convert everything to UTC. + # MVT should output the local time only: + # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000 + assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.000000"