diff --git a/pyproject.toml b/pyproject.toml index 96971b7..eec0824 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "betterproto >=1.2.0", "pydantic >= 2.10.0", "pydantic-settings >= 2.7.0", + 'backports.zoneinfo; python_version < "3.9"', ] requires-python = ">= 3.8" diff --git a/src/mvt/android/artifacts/file_timestamps.py b/src/mvt/android/artifacts/file_timestamps.py new file mode 100644 index 0000000..aa2dc25 --- /dev/null +++ b/src/mvt/android/artifacts/file_timestamps.py @@ -0,0 +1,43 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ +from typing import Union + +from .artifact import AndroidArtifact + + +class FileTimestampsArtifact(AndroidArtifact): + def serialize(self, record: dict) -> Union[dict, list]: + records = [] + + for ts in set( + [ + record.get("access_time"), + record.get("changed_time"), + record.get("modified_time"), + ] + ): + if not ts: + continue + + macb = "" + macb += "M" if ts == record.get("modified_time") else "-" + macb += "A" if ts == record.get("access_time") else "-" + macb += "C" if ts == record.get("changed_time") else "-" + macb += "-" + + msg = record["path"] + if record.get("context"): + msg += f" ({record['context']})" + + records.append( + { + "timestamp": ts, + "module": self.__class__.__name__, + "event": macb, + "data": msg, + } + ) + + return records diff --git a/src/mvt/android/artifacts/getprop.py b/src/mvt/android/artifacts/getprop.py index a4d87e0..6c7030f 100644 --- a/src/mvt/android/artifacts/getprop.py +++ b/src/mvt/android/artifacts/getprop.py @@ -42,6 +42,17 @@ class GetProp(AndroidArtifact): entry = {"name": matches[0][0], "value": matches[0][1]} self.results.append(entry) + def get_device_timezone(self) -> str: + """ + Get the device timezone from the getprop results + + Used in other moduels to calculate the timezone offset + """ + for entry in self.results: + if entry["name"] == "persist.sys.timezone": + return entry["value"] + return None + def check_indicators(self) -> None: for entry in self.results: if entry["name"] in INTERESTING_PROPERTIES: diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index 62c329a..3827154 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -81,8 +81,8 @@ class TombstoneCrashArtifact(AndroidArtifact): "module": self.__class__.__name__, "event": "Tombstone", "data": ( - f"Crash in '{record['process_name']}' process running as UID '{record['uid']}' at " - f"{record['timestamp']}. Crash type '{record['signal_info']['name']}' with code '{record['signal_info']['code_name']}'" + f"Crash in '{record['process_name']}' process running as UID '{record['uid']}' in file '{record['file_name']}' " + f"Crash type '{record['signal_info']['name']}' with code '{record['signal_info']['code_name']}'" ), } @@ -258,7 +258,10 @@ class TombstoneCrashArtifact(AndroidArtifact): timestamp_parsed = datetime.datetime.strptime( timestamp_without_micro, "%Y-%m-%d %H:%M:%S%z" ) - return convert_datetime_to_iso(timestamp_parsed) + + # HACK: Swap the local timestamp to UTC, so keep the original time and avoid timezone conversion. + local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc) + return convert_datetime_to_iso(local_timestamp) @staticmethod def _proccess_name_from_thread(tombstone_dict: dict) -> str: diff --git a/src/mvt/android/modules/androidqf/base.py b/src/mvt/android/modules/androidqf/base.py index 4e2c68e..d871059 100644 --- a/src/mvt/android/modules/androidqf/base.py +++ b/src/mvt/android/modules/androidqf/base.py @@ -48,6 +48,37 @@ class AndroidQFModule(MVTModule): def _get_files_by_pattern(self, pattern: str): return fnmatch.filter(self.files, pattern) + def _get_device_timezone(self): + """ + Get the device timezone from the getprop.txt file. + + This is needed to map local timestamps stored in some + Android log files to UTC/timezone-aware timestamps. + """ + get_prop_files = self._get_files_by_pattern("*/getprop.txt") + if not get_prop_files: + self.log.warning( + "Could not find getprop.txt file. " + "Some timestamps and timeline data may be incorrect." + ) + return None + + from mvt.android.artifacts.getprop import GetProp + + properties_artifact = GetProp() + prop_data = self._get_file_content(get_prop_files[0]).decode("utf-8") + properties_artifact.parse(prop_data) + timezone = properties_artifact.get_device_timezone() + if timezone: + self.log.debug("Identified local phone timezone: %s", timezone) + return timezone + + self.log.warning( + "Could not find or determine local device timezone. " + "Some timestamps and timeline data may be incorrect." + ) + return None + def _get_file_content(self, file_path): if self.archive: handle = self.archive.open(file_path) diff --git a/src/mvt/android/modules/androidqf/files.py b/src/mvt/android/modules/androidqf/files.py index fde7ca1..22b832c 100644 --- a/src/mvt/android/modules/androidqf/files.py +++ b/src/mvt/android/modules/androidqf/files.py @@ -6,6 +6,11 @@ import datetime import json import logging + +try: + import zoneinfo +except ImportError: + from backports import zoneinfo from typing import Optional, Union from mvt.android.modules.androidqf.base import AndroidQFModule @@ -106,6 +111,12 @@ class Files(AndroidQFModule): # TODO: adds SHA1 and MD5 when available in MVT def run(self) -> None: + if timezone := self._get_device_timezone(): + device_timezone = zoneinfo.ZoneInfo(timezone) + else: + self.log.warning("Unable to determine device timezone, using UTC") + device_timezone = zoneinfo.ZoneInfo("UTC") + for file in self._get_files_by_pattern("*/files.json"): rawdata = self._get_file_content(file).decode("utf-8", errors="ignore") try: @@ -120,11 +131,18 @@ class Files(AndroidQFModule): for file_data in data: for ts in ["access_time", "changed_time", "modified_time"]: if ts in file_data: - file_data[ts] = convert_datetime_to_iso( - datetime.datetime.fromtimestamp( - file_data[ts], tz=datetime.timezone.utc - ) + utc_timestamp = datetime.datetime.fromtimestamp( + file_data[ts], tz=datetime.timezone.utc ) + # Convert the UTC timestamp to local tiem on Android device's local timezone + local_timestamp = utc_timestamp.astimezone(device_timezone) + + # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we + # set the timestamp timezone to UTC, to avoid the timezone conversion again. + local_timestamp = local_timestamp.replace( + tzinfo=datetime.timezone.utc + ) + file_data[ts] = convert_datetime_to_iso(local_timestamp) self.results.append(file_data) diff --git a/src/mvt/android/modules/androidqf/logfile_timestamps.py b/src/mvt/android/modules/androidqf/logfile_timestamps.py new file mode 100644 index 0000000..b37851d --- /dev/null +++ b/src/mvt/android/modules/androidqf/logfile_timestamps.py @@ -0,0 +1,65 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import os +import datetime +import logging +from typing import Optional + +from mvt.common.utils import convert_datetime_to_iso +from .base import AndroidQFModule +from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact + + +class LogsFileTimestamps(FileTimestampsArtifact, AndroidQFModule): + """This module extracts records from battery daily updates.""" + + slug = "logfile_timestamps" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def _get_file_modification_time(self, file_path: str) -> dict: + if self.archive: + file_timetuple = self.archive.getinfo(file_path).date_time + return datetime.datetime(*file_timetuple) + else: + file_stat = os.stat(os.path.join(self.parent_path, file_path)) + return datetime.datetime.fromtimestamp(file_stat.st_mtime) + + def run(self) -> None: + filesystem_files = self._get_files_by_pattern("*/logs/*") + + self.results = [] + for file in filesystem_files: + # Only the modification time is available in the zip file metadata. + # The timezone is the local timezone of the machine the phone. + modification_time = self._get_file_modification_time(file) + self.results.append( + { + "path": file, + "modified_time": convert_datetime_to_iso(modification_time), + } + ) + + self.log.info( + "Extracted a total of %d filesystem timestamps from AndroidQF logs directory.", + len(self.results), + ) diff --git a/src/mvt/android/modules/bugreport/__init__.py b/src/mvt/android/modules/bugreport/__init__.py index 6945629..b5a1247 100644 --- a/src/mvt/android/modules/bugreport/__init__.py +++ b/src/mvt/android/modules/bugreport/__init__.py @@ -14,6 +14,7 @@ from .packages import Packages from .platform_compat import PlatformCompat from .receivers import Receivers from .adb_state import DumpsysADBState +from .fs_timestamps import BugReportTimestamps from .tombstones import Tombstones BUGREPORT_MODULES = [ @@ -28,5 +29,6 @@ BUGREPORT_MODULES = [ PlatformCompat, Receivers, DumpsysADBState, + BugReportTimestamps, Tombstones, ] diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index bf98dca..77802b2 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -6,6 +6,7 @@ import datetime import fnmatch import logging import os + from typing import List, Optional from zipfile import ZipFile diff --git a/src/mvt/android/modules/bugreport/fs_timestamps.py b/src/mvt/android/modules/bugreport/fs_timestamps.py new file mode 100644 index 0000000..14e1cd1 --- /dev/null +++ b/src/mvt/android/modules/bugreport/fs_timestamps.py @@ -0,0 +1,55 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +import logging +from typing import Optional + +from mvt.common.utils import convert_datetime_to_iso +from .base import BugReportModule +from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact + + +class BugReportTimestamps(FileTimestampsArtifact, BugReportModule): + """This module extracts records from battery daily updates.""" + + slug = "bugreport_timestamps" + + def __init__( + self, + file_path: Optional[str] = None, + target_path: Optional[str] = None, + results_path: Optional[str] = None, + module_options: Optional[dict] = None, + log: logging.Logger = logging.getLogger(__name__), + results: Optional[list] = None, + ) -> None: + super().__init__( + file_path=file_path, + target_path=target_path, + results_path=results_path, + module_options=module_options, + log=log, + results=results, + ) + + def run(self) -> None: + filesystem_files = self._get_files_by_pattern("FS/*") + + self.results = [] + for file in filesystem_files: + # Only the modification time is available in the zip file metadata. + # The timezone is the local timezone of the machine the phone. + modification_time = self._get_file_modification_time(file) + self.results.append( + { + "path": file, + "modified_time": convert_datetime_to_iso(modification_time), + } + ) + + self.log.info( + "Extracted a total of %d filesystem timestamps from bugreport.", + len(self.results), + ) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index 01309b8..7f65843 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -101,15 +101,25 @@ class Command: if not self.results_path: return + # We use local timestamps in the timeline on Android as many + # logs do not contain timezone information. + if type(self).__name__.startswith("CmdAndroid"): + is_utc = False + else: + is_utc = True + if len(self.timeline) > 0: save_timeline( - self.timeline, os.path.join(self.results_path, "timeline.csv") + self.timeline, + os.path.join(self.results_path, "timeline.csv"), + is_utc=is_utc, ) if len(self.timeline_detected) > 0: save_timeline( self.timeline_detected, os.path.join(self.results_path, "timeline_detected.csv"), + is_utc=is_utc, ) def _store_info(self) -> None: diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index 03e504f..1468cf4 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -227,7 +227,7 @@ def run_module(module: MVTModule) -> None: module.save_to_json() -def save_timeline(timeline: list, timeline_path: str) -> None: +def save_timeline(timeline: list, timeline_path: str, is_utc: bool = True) -> None: """Save the timeline in a csv file. :param timeline: List of records to order and store @@ -238,7 +238,12 @@ def save_timeline(timeline: list, timeline_path: str) -> None: csvoutput = csv.writer( handle, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, escapechar="\\" ) - csvoutput.writerow(["UTC Timestamp", "Plugin", "Event", "Description"]) + + if is_utc: + timestamp_header = "UTC Timestamp" + else: + timestamp_header = "Device Local Timestamp" + csvoutput.writerow([timestamp_header, "Plugin", "Event", "Description"]) for event in sorted( timeline, key=lambda x: x["timestamp"] if x["timestamp"] is not None else "" diff --git a/tests/android/test_artifact_tombstones.py b/tests/android/test_artifact_tombstones.py index b837d73..487eb21 100644 --- a/tests/android/test_artifact_tombstones.py +++ b/tests/android/test_artifact_tombstones.py @@ -60,6 +60,8 @@ class TestTombstoneCrashArtifact: assert tombstone_result.get("pid") == 25541 assert tombstone_result.get("process_name") == "mtk.ape.decoder" - # Check if the timestamp is correctly parsed, and converted to UTC - # Original is in +0200: 2023-04-12 12:32:40.518290770+0200, result should be 2023-04-12 10:32:40.000000+0000 - assert tombstone_result.get("timestamp") == "2023-04-12 10:32:40.000000" + # With Android logs we want to keep timestamps as device local time for consistency. + # We often don't know the time offset for a log entry and so can't convert everything to UTC. + # MVT should output the local time only: + # So original 2023-04-12 12:32:40.518290770+0200 -> 2023-04-12 12:32:40.000000 + assert tombstone_result.get("timestamp") == "2023-04-12 12:32:40.000000"