Read bugreport file timestamps in the device's timezone (#921)

A bugreport zip stores each entry's time as the device's wall clock, and
an unpacked bugreport's mtimes are whatever the extraction left. The
bugreport modules read both as naive local datetimes, so a tombstone's
file_timestamp moved with the analysing machine's timezone and sat three
hours from the crash time the tombstone itself records for a device in
Nairobi.

check-bugreport now resolves the device's timezone once, from --timezone
or from persist.sys.timezone in the dumpstate's SYSTEM PROPERTIES, and
hands it to the modules as device_timezone the way check-androidqf does;
a zone already known, as when androidqf drives the bugreport inside its
own archive, is kept. Zip entry times are read in that zone and the
mtimes of an unpacked bugreport as UTC instants, so convert_datetime_to_iso
writes both in UTC. Without a zone the wall clock is kept naive and a
warning says so, and an unpacked bugreport gets a warning that its file
timestamps are the extraction's, not the device's. BugReportTimestamps
uses the same reading instead of parsing the properties itself.

src/mvt/android/cli.py

@@ -150,6 +150,16 @@ def check_adb(ctx):
     default=[],
     help=HELP_MSG_LOAD_MODULE,
 )
+@click.option(
+    "--timezone",
+    "-t",
+    default=None,
+    help=(
+        "IANA timezone name for the device, for example 'Europe/Paris'. "
+        "Bugreport file timestamps are the device's wall clock; by default the "
+        "zone is read from persist.sys.timezone in the bugreport itself."
+    ),
+)
 @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE_COMMAND)
 @click.argument("BUGREPORT_PATH", type=click.Path(exists=True))
 @click.pass_context
@@ -160,6 +170,7 @@ def check_bugreport(
     list_modules,
     module,
     load_module,
+    timezone,
     verbose,
     bugreport_path,
 ):
@@ -167,12 +178,18 @@ def check_bugreport(
 
     set_verbose_logging(verbose or _get_verbose(ctx))
     custom_modules = _load_custom_modules(load_module)
+
+    module_options = {}
+    if timezone:
+        module_options["device_timezone"] = timezone
+
     # Always generate hashes as bug reports are small.
     cmd = CmdAndroidCheckBugreport(
         target_path=bugreport_path,
         results_path=output,
         ioc_files=iocs,
         module_name=module,
+        module_options=module_options if module_options else None,
         hashes=True,
         disable_version_check=_get_disable_flags(ctx)[0],
         disable_indicator_check=_get_disable_flags(ctx)[1],

src/mvt/android/cmd_check_bugreport.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from typing import List, Optional
 from zipfile import ZipFile
 
+from mvt.android.artifacts.getprop import GetProp
 from mvt.android.modules.bugreport.base import BugReportModule
 from mvt.common.command import Command
 from mvt.common.indicators import Indicators
@@ -88,13 +89,56 @@ class CmdAndroidCheckBugreport(Command):
             self.__files.append(file_name)
 
     def init(self) -> None:
-        if not self.target_path:
+        if self.target_path:
+            if os.path.isfile(self.target_path):
+                self.from_zip(ZipFile(self.target_path))
+            elif os.path.isdir(self.target_path):
+                self.from_dir(self.target_path)
+                self.log.warning(
+                    "Analysing an unpacked bugreport: file timestamps come from "
+                    "the extraction, not from the device. Analyse the original "
+                    "zip to keep the device's file timestamps."
+                )
+        if self.__format:
+            self._resolve_device_timezone()
+
+    def _resolve_device_timezone(self) -> None:
+        """Name the device's timezone in module_options unless it is known already.
+
+        A bugreport's SYSTEM PROPERTIES section carries persist.sys.timezone.
+        Zip entry times are the device's wall clock, and modules read them in
+        this zone; --timezone or check-androidqf's own reading takes precedence.
+        """
+        if self.module_options.get("device_timezone"):
+            self.log.info("Device timezone: %s", self.module_options["device_timezone"])
             return
 
-        if os.path.isfile(self.target_path):
-            self.from_zip(ZipFile(self.target_path))
-        elif os.path.isdir(self.target_path):
-            self.from_dir(self.target_path)
+        probe = BugReportModule(log=self.log)
+        self.module_init(probe)
+        timezone = None
+        try:
+            dumpstate = probe._get_dumpstate_file()
+        except Exception as exc:
+            self.log.warning("Could not read the bugreport's dumpstate: %s", exc)
+            dumpstate = None
+        if dumpstate:
+            properties = GetProp()
+            properties.parse(
+                BugReportModule.extract_command_section(
+                    dumpstate.decode("utf-8", errors="replace"),
+                    "------ SYSTEM PROPERTIES",
+                )
+            )
+            timezone = properties.get_device_timezone()
+        if timezone:
+            self.log.info("Device timezone identified from the bugreport: %s", timezone)
+            self.module_options["device_timezone"] = timezone
+        else:
+            self.log.warning(
+                "persist.sys.timezone not found in the bugreport; file timestamps "
+                "are the device's wall clock without a timezone. Pass --timezone "
+                "to name it."
+            )
 
     def module_init(self, module: BugReportModule) -> None:  # type: ignore[override]
         if self.__format == "zip":

src/mvt/android/modules/bugreport/base.py

@@ -9,6 +9,7 @@ import os
 from pathlib import Path
 from typing import List, Optional
 from zipfile import ZipFile
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from mvt.common.module import ModuleResults, MVTModule
 
@@ -125,12 +126,31 @@ class BugReportModule(MVTModule):
             lines.append(line)
         return "\n".join(lines)
 
+    def _device_timezone(self) -> Optional[datetime.tzinfo]:
+        """The device's timezone named in module_options, or None when unknown."""
+        name = self.module_options.get("device_timezone")
+        if not name:
+            return None
+        try:
+            return ZoneInfo(name)
+        except ZoneInfoNotFoundError:
+            self.log.warning("Unknown device timezone %s", name)
+            return None
+
     def _get_file_modification_time(self, file_path: str) -> datetime.datetime:
+        """When the file was last modified.
+
+        A zip entry carries the device's wall clock, so it is returned in the
+        device's timezone when the bugreport names one and naive otherwise.
+        An unpacked bugreport's mtime is whatever the extraction left, an
+        instant returned in UTC.
+        """
         if self.zip_archive:
             file_timetuple = self.zip_archive.getinfo(file_path).date_time
-            return datetime.datetime(*file_timetuple)
-        else:
-            if not self.extract_path:
-                raise ValueError("extract_path is not set")
-            file_stat = os.stat(os.path.join(self.extract_path, file_path))
-            return datetime.datetime.fromtimestamp(file_stat.st_mtime)
+            return datetime.datetime(*file_timetuple, tzinfo=self._device_timezone())
+        if not self.extract_path:
+            raise ValueError("extract_path is not set")
+        file_stat = os.stat(os.path.join(self.extract_path, file_path))
+        return datetime.datetime.fromtimestamp(
+            file_stat.st_mtime, tz=datetime.timezone.utc
+        )

src/mvt/android/modules/bugreport/fs_timestamps.py

@@ -4,15 +4,12 @@
 #   https://license.mvt.re/1.1/
 
 import logging
-import datetime
 from typing import Optional
-from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from mvt.common.utils import convert_datetime_to_iso
 from .base import BugReportModule
 from mvt.common.module_types import ModuleResults
 from mvt.android.artifacts.file_timestamps import FileTimestampsArtifact
-from mvt.android.artifacts.getprop import GetProp
 
 
 class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
@@ -41,41 +38,28 @@ class BugReportTimestamps(FileTimestampsArtifact, BugReportModule):
     def run(self) -> None:
         filesystem_files = self._get_files_by_pattern("FS/*")
 
-        timezone_name = None
-        dumpstate = self._get_dumpstate_file()
-        if dumpstate:
-            section = self.extract_command_section(
-                dumpstate.decode("utf-8", errors="replace"),
-                "------ SYSTEM PROPERTIES",
-            )
-            properties = GetProp()
-            properties.parse(section)
-            timezone_name = properties.get_device_timezone()
-        timezone = None
-        if timezone_name:
-            try:
-                timezone = ZoneInfo(timezone_name)
-            except ZoneInfoNotFoundError:
-                self.log.warning("Unknown device timezone %s", timezone_name)
-
         self.results = []
         for file in filesystem_files:
-            # Only the modification time is available in the zip file metadata.
-            # The timezone is the local timezone of the machine the phone.
+            # A zip entry keeps the device's wall clock, read in the device's
+            # timezone when the command found one (see CmdAndroidCheckBugreport);
+            # an unpacked bugreport's mtime is read as a UTC instant.
             modification_time = self._get_file_modification_time(file)
-            utc_time = None
-            if timezone is not None:
-                utc_time = convert_datetime_to_iso(
-                    modification_time.replace(tzinfo=timezone).astimezone(
-                        datetime.timezone.utc
-                    )
-                )
             self.results.append(
                 {
                     "path": file,
-                    "modified_time": convert_datetime_to_iso(modification_time),
-                    "modified_time_utc": utc_time,
-                    "timezone": timezone_name,
+                    "modified_time": convert_datetime_to_iso(
+                        modification_time.replace(tzinfo=None)
+                    ),
+                    "modified_time_utc": (
+                        convert_datetime_to_iso(modification_time)
+                        if modification_time.tzinfo
+                        else None
+                    ),
+                    "timezone": (
+                        self.module_options.get("device_timezone")
+                        if self.zip_archive
+                        else "UTC"
+                    ),
                     "timestamp_source": (
                         "zip_metadata" if self.zip_archive else "filesystem_metadata"
                     ),

tests/test_check_android_bugreport.py

@@ -3,11 +3,16 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
+import datetime
+import logging
 import os
+import shutil
+import zipfile
 
 from click.testing import CliRunner
 
 from mvt.android.cli import check_bugreport
+from mvt.android.cmd_check_bugreport import CmdAndroidCheckBugreport
 
 from .utils import get_artifact_folder
 
@@ -28,3 +33,90 @@ class TestCheckBugreportCommand:
         assert result.exit_code == 1
         assert "Invalid bugreport archive" in result.output
         assert "Traceback" not in result.output
+
+
+PROPERTIES = (
+    "------ SYSTEM PROPERTIES (getprop) ------\n"
+    "[persist.sys.timezone]: [Africa/Nairobi]\n"
+    "------ 0.01s was the duration of 'SYSTEM PROPERTIES' ------\n"
+)
+TOMBSTONE = "android_data/bugreport/FS/data/tombstones/tombstone_00"
+
+
+def _bugreport_zip(tmp_path, dumpstate=PROPERTIES):
+    """A bugreport zip holding one tombstone written at 11:38:10 device time.
+
+    An even second: zip entry times have a two-second resolution.
+    """
+    path = tmp_path / "bugreport.zip"
+    with open(os.path.join(get_artifact_folder(), TOMBSTONE), "rb") as handle:
+        tombstone = handle.read()
+    with zipfile.ZipFile(path, "w") as archive:
+        archive.writestr("main_entry.txt", "dumpstate.txt")
+        archive.writestr("dumpstate.txt", dumpstate)
+        entry = zipfile.ZipInfo(
+            "FS/data/tombstones/tombstone_00", date_time=(2023, 3, 10, 11, 38, 10)
+        )
+        archive.writestr(entry, tombstone)
+    return str(path)
+
+
+def _tombstone_timestamp(target, **options):
+    cmd = CmdAndroidCheckBugreport(
+        target_path=target,
+        module_name="Tombstones",
+        disable_version_check=True,
+        disable_indicator_check=True,
+        **options,
+    )
+    cmd.run()
+    return cmd, cmd.executed[0].results[0]["file_timestamp"]
+
+
+class TestCheckBugreportTimezone:
+    def test_zip_entry_times_are_read_in_the_device_timezone(self, tmp_path):
+        cmd, file_timestamp = _tombstone_timestamp(_bugreport_zip(tmp_path))
+
+        assert cmd.module_options["device_timezone"] == "Africa/Nairobi"
+        # 11:38:10 in Nairobi is 08:38:10 UTC.
+        assert file_timestamp == "2023-03-10 08:38:10.000000"
+
+    def test_timezone_option_wins_over_the_bugreport(self, tmp_path):
+        _, file_timestamp = _tombstone_timestamp(
+            _bugreport_zip(tmp_path), module_options={"device_timezone": "Europe/Paris"}
+        )
+
+        assert file_timestamp == "2023-03-10 10:38:10.000000"
+
+        result = CliRunner().invoke(
+            check_bugreport,
+            ["-t", "Europe/Paris", "-m", "Tombstones", _bugreport_zip(tmp_path)],
+        )
+        assert result.exit_code == 0, result.output
+
+    def test_without_a_timezone_the_wall_clock_is_kept_and_a_warning_given(
+        self, tmp_path, caplog
+    ):
+        with caplog.at_level(logging.WARNING, logger="mvt"):
+            _, file_timestamp = _tombstone_timestamp(_bugreport_zip(tmp_path, ""))
+
+        assert file_timestamp == "2023-03-10 11:38:10.000000"
+        assert "persist.sys.timezone not found" in caplog.text
+
+    def test_unpacked_bugreport_warns_and_reads_mtimes_as_utc(self, tmp_path, caplog):
+        unpacked = tmp_path / "bugreport"
+        shutil.copytree(
+            os.path.join(get_artifact_folder(), "android_data/bugreport"), unpacked
+        )
+        instant = datetime.datetime(
+            2023, 3, 10, 8, 38, 11, tzinfo=datetime.timezone.utc
+        ).timestamp()
+        for name in ("tombstone_00", "tombstone_01"):
+            os.utime(unpacked / "FS" / "data" / "tombstones" / name, (instant, instant))
+
+        with caplog.at_level(logging.WARNING, logger="mvt"):
+            _, file_timestamp = _tombstone_timestamp(str(unpacked))
+
+        assert "unpacked bugreport" in caplog.text
+        # Whatever the zone of the machine running the analysis.
+        assert file_timestamp == "2023-03-10 08:38:11.000000"