Merge branch 'main' into fix/settings-env-persist

src/mvt/android/artifacts/settings.py

@@ -4,6 +4,11 @@
 #   https://license.mvt.re/1.1/
 
 import re
+from datetime import datetime
+from typing import Optional, Sequence
+
+from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult
+from mvt.common.utils import convert_datetime_to_iso
 
 from .artifact import AndroidArtifact
 
@@ -60,45 +65,240 @@ ANDROID_DANGEROUS_SETTINGS = [
     },
 ]
 
+# dumpsys prints the fields of a setting record, and of a change history entry,
+# always in this order and separated by a single space. After the value come
+# `default:` and `defaultSystemSet:` when a default is recorded, then `tag:`;
+# some vendor builds add whether the value survives a restore, either as
+# `isValuePreservedInRestore:` or as a bare `notPreservedInRestore` token.
+SETTING_FIELDS = (
+    "_id",
+    "name",
+    "pkg",
+    "value",
+    "default",
+    "defaultSystemSet",
+    "tag",
+    "isValuePreservedInRestore",
+)
+HISTORY_FIELDS = ("time", "mode", "oldValue", "newValue", "package")
+
+NAMESPACE_PATTERN = re.compile(
+    r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$"
+)
+SECTION_END_PATTERN = re.compile(r"ending at: (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
+
 
 class Settings(AndroidArtifact):
-    def parse(self, content: str) -> None:
-        self.results: dict[str, dict[str, str]] = {}
-        namespace: str | None = None
-        for line in content.splitlines():
-            heading = re.match(
-                r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$",
-                line.strip(),
-            )
-            if heading:
-                namespace = f"{heading.group(1).lower()}:user_{heading.group(2)}"
-                self.results[namespace] = {}
+    """Parser for the `dumpsys settings` output.
+
+    Every row of the settings provider becomes one result, keeping the fields
+    dumpsys prints alongside the value: the row id, the package which recorded
+    the setting, the default, the tag, and the change history. A setting name
+    can appear more than once within a namespace, so results are a list rather
+    than a mapping.
+    """
+
+    def serialize(self, result: ModuleAtomicResult) -> ModuleSerializedResult:
+        records = []
+        for entry in result.get("history", []):
+            if not entry.get("timestamp"):
                 continue
-            if namespace is None or not line.startswith("_id:"):
-                continue
-            setting = re.match(
-                r"^_id:\S+\s+name:(.*?)\s+pkg:.*?\s+value:(.*?)"
-                r"(?:\s+default:.*\s+defaultSystemSet:(?:true|false))?$",
-                line,
+
+            records.append(
+                {
+                    "timestamp": entry["timestamp"],
+                    "module": self.__class__.__name__,
+                    "event": "settings_change",
+                    "data": (
+                        f"{result.get('namespace')} setting "
+                        f'"{result.get("name")}" changed from '
+                        f'"{entry.get("oldValue")}" to "{entry.get("newValue")}" '
+                        f"by {entry.get('pkg')}"
+                    ),
+                }
             )
-            if setting:
-                self.results[namespace][setting.group(1)] = setting.group(2)
+
+        return records
 
     def check_indicators(self) -> None:
-        for namespace, settings in self.results.items():
-            for key, value in settings.items():
-                for danger in ANDROID_DANGEROUS_SETTINGS:
-                    # Check if one of the dangerous settings is using an unsafe
-                    # value (different than the one specified).
-                    if danger["key"] == key and danger["safe_value"] != value:
-                        self.alertstore.medium(
-                            f'Found suspicious "{namespace}" setting "{key} = {value}" ({danger["description"]})',
-                            "",
-                            {
-                                "namespace": namespace,
-                                "key": key,
-                                "value": value,
-                                "description": danger["description"],
-                            },
-                        )
-                        break
+        for result in self.results:
+            name = result.get("name")
+            value = result.get("value")
+            for danger in ANDROID_DANGEROUS_SETTINGS:
+                # Check if one of the dangerous settings is using an unsafe
+                # value (different than the one specified).
+                if danger["key"] != name or danger["safe_value"] == value:
+                    continue
+
+                history = result.get("history") or []
+                self.alertstore.medium(
+                    f'Found suspicious "{result.get("namespace")}" setting '
+                    f'"{name} = {value}" ({danger["description"]})',
+                    history[-1]["timestamp"] if history else "",
+                    result,
+                )
+                break
+
+    def parse(self, content: str) -> None:
+        self.results: list[ModuleAtomicResult] = []
+        section_end = self._parse_section_end(content)
+        namespace: Optional[str] = None
+        user: Optional[str] = None
+        record_lines: list[str] = []
+        history_lines: list[str] = []
+        in_history = False
+
+        def flush() -> None:
+            nonlocal record_lines, history_lines, in_history
+            if record_lines:
+                self.results.append(
+                    self._build_record(
+                        namespace, user, record_lines, history_lines, section_end
+                    )
+                )
+            record_lines = []
+            history_lines = []
+            in_history = False
+
+        for line in content.splitlines():
+            heading = NAMESPACE_PATTERN.match(line.strip())
+            if heading:
+                flush()
+                namespace = heading.group(1).lower()
+                user = heading.group(2)
+                continue
+
+            if line.startswith("--------- "):
+                # dumpsys closes every section with a duration trailer.
+                flush()
+                namespace = None
+                continue
+
+            if namespace is None:
+                continue
+
+            if not line.strip():
+                # dumpsys prints a blank line after every namespace block and
+                # after a change history, and other dumps such as the
+                # generation registry follow the last block, so a blank line
+                # closes the record being read.
+                flush()
+                continue
+
+            if line.startswith("_id:"):
+                flush()
+                record_lines = [line]
+                continue
+
+            if not record_lines:
+                continue
+
+            stripped = line.strip()
+            if stripped.startswith("History ("):
+                in_history = True
+                continue
+
+            if in_history:
+                if stripped.startswith("time:"):
+                    history_lines.append(stripped)
+                elif stripped and history_lines:
+                    # A history entry can be wrapped over several lines.
+                    history_lines[-1] += " " + stripped
+                continue
+
+            # Anything else continues the value of the record being read.
+            record_lines.append(line)
+
+        flush()
+
+    @staticmethod
+    def _split_fields(text: str, keys: Sequence[str]) -> dict[str, str]:
+        """Split the `key:value` fields of one record.
+
+        Values are free-form and may contain spaces and newlines, so a field
+        runs up to the start of the next key which is actually present. Keys
+        dumpsys did not print are skipped.
+        """
+        fields: dict[str, str] = {}
+        key = keys[0]
+        if not text.startswith(f"{key}:"):
+            return fields
+
+        remainder = text[len(key) + 1 :]
+        for next_key in keys[1:]:
+            value, separator, rest = remainder.partition(f" {next_key}:")
+            if separator:
+                fields[key] = value
+                key, remainder = next_key, rest
+
+        fields[key] = remainder
+        return fields
+
+    @staticmethod
+    def _parse_section_end(content: str) -> Optional[datetime]:
+        """Return the time the settings section was dumped, if reported."""
+        match = SECTION_END_PATTERN.search(content)
+        if not match:
+            return None
+
+        try:
+            return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            return None
+
+    @staticmethod
+    def _resolve_timestamp(
+        value: str, section_end: Optional[datetime]
+    ) -> Optional[str]:
+        """Add the missing year to a `MM-DD HH:MM:SS.mmm` history timestamp.
+
+        dumpsys prints the change history without a year, so it is resolved
+        against the time the section was dumped: the most recent matching date
+        at or before that time.
+        """
+        if section_end is None:
+            return None
+
+        try:
+            partial = datetime.strptime(value, "%m-%d %H:%M:%S.%f")
+            timestamp = partial.replace(year=section_end.year)
+            if timestamp > section_end:
+                timestamp = partial.replace(year=section_end.year - 1)
+        except ValueError:
+            return None
+
+        return convert_datetime_to_iso(timestamp)
+
+    def _parse_history(
+        self, line: str, section_end: Optional[datetime]
+    ) -> ModuleAtomicResult:
+        fields = self._split_fields(line, HISTORY_FIELDS)
+        return {
+            "timestamp": self._resolve_timestamp(fields.get("time", ""), section_end),
+            "oldValue": fields.get("oldValue"),
+            "newValue": fields.get("newValue"),
+            "pkg": fields.get("package"),
+        }
+
+    def _build_record(
+        self,
+        namespace: Optional[str],
+        user: Optional[str],
+        record_lines: list[str],
+        history_lines: list[str],
+        section_end: Optional[datetime],
+    ) -> ModuleAtomicResult:
+        text = "\n".join(record_lines).rstrip()
+        # The bare `notPreservedInRestore` token has no `key:` shape and is
+        # printed last, so peel it off before splitting the fields.
+        head = text.removesuffix(" notPreservedInRestore")
+
+        record: ModuleAtomicResult = {"namespace": namespace, "user": user}
+        record.update(self._split_fields(head, SETTING_FIELDS))
+        if head != text:
+            record["isValuePreservedInRestore"] = "false"
+
+        record["history"] = [
+            self._parse_history(entry, section_end) for entry in history_lines
+        ]
+        return record

src/mvt/android/modules/androidqf/aqf_settings.py

@@ -3,11 +3,7 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import logging
-from typing import Optional
-
 from mvt.android.artifacts.settings import Settings as SettingsArtifact
-from mvt.common.module_types import ModuleResults
 
 from .base import AndroidQFModule
 
@@ -15,43 +11,23 @@ from .base import AndroidQFModule
 class AQFSettings(SettingsArtifact, AndroidQFModule):
     """This module analyse setting files"""
 
-    def __init__(
-        self,
-        file_path: Optional[str] = None,
-        target_path: Optional[str] = None,
-        results_path: Optional[str] = None,
-        module_options: Optional[dict] = None,
-        log: logging.Logger = logging.getLogger(__name__),
-        results: Optional[ModuleResults] = None,
-    ) -> None:
-        super().__init__(
-            file_path=file_path,
-            target_path=target_path,
-            results_path=results_path,
-            module_options=module_options,
-            log=log,
-            results=results,
-        )
-        self.results: dict = results if results is not None else {}
-
     def run(self) -> None:
         for setting_file in self._get_files_by_pattern("*/settings_*.txt"):
             namespace = setting_file[setting_file.rfind("_") + 1 : -4]
 
-            self.results[namespace] = {}
             data = self._get_file_content(setting_file)
             for line in data.decode("utf-8").splitlines():
-                line = line.strip()
-                try:
-                    key, value = line.split("=", 1)
-                except ValueError:
+                name, separator, value = line.strip().partition("=")
+                if not separator:
                     continue
 
-                try:
-                    self.results[namespace][key] = value
-                except IndexError:
-                    continue
+                self.results.append(
+                    {
+                        "namespace": namespace,
+                        "user": None,
+                        "name": name,
+                        "value": value,
+                    }
+                )
 
-        self.log.info(
-            "Identified %d settings", sum([len(val) for val in self.results.values()])
-        )
+        self.log.info("Identified %d settings", len(self.results))

src/mvt/android/modules/bugreport/settings.py

@@ -18,5 +18,4 @@ class Settings(SettingsArtifact, BugReportModule):
             data.decode("utf-8", errors="replace"), "DUMP OF SERVICE settings:"
         )
         self.parse(section)
-        count = sum(len(settings) for settings in self.results.values())
-        self.log.info("Identified %d Android settings", count)
+        self.log.info("Identified %d Android settings", len(self.results))

tests/android/test_artifact_settings.py

@@ -0,0 +1,170 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+
+from mvt.android.artifacts.settings import Settings
+
+from ..utils import get_artifact
+
+
+def parse_bugreport_settings() -> Settings:
+    settings = Settings()
+    with open(get_artifact("android_data/bugreport/dumpstate.txt")) as handle:
+        data = handle.read()
+
+    settings.parse(settings.extract_dumpsys_section(data, "DUMP OF SERVICE settings:"))
+    return settings
+
+
+def find(settings: Settings, name: str) -> list:
+    return [result for result in settings.results if result["name"] == name]
+
+
+class TestSettingsArtifact:
+    def test_parsing(self):
+        settings = parse_bugreport_settings()
+
+        assert len(settings.results) == 12
+        assert {result["namespace"] for result in settings.results} == {
+            "config",
+            "global",
+            "secure",
+        }
+        assert settings.results[0] == {
+            "namespace": "config",
+            "user": "0",
+            "_id": "682",
+            "name": "namespace_one/blocked_components",
+            "pkg": "com.example.services",
+            "value": (
+                "com.android.settings,com.android.vending,\n"
+                "com.example.dialer,\n"
+                "com.example.camera"
+            ),
+            "default": (
+                "com.android.settings,\n"
+                "        com.android.vending,\n"
+                "        com.example.dialer"
+            ),
+            "defaultSystemSet": "false",
+            "history": [],
+        }
+
+    def test_multiline_values_are_kept_whole(self):
+        settings = parse_bugreport_settings()
+
+        assert find(settings, "namespace_one/allowed_packages")[0]["value"] == (
+            "com.example.messaging,\ncom.example.chat"
+        )
+        assert find(settings, "widget_instance_data")[0]["value"] == (
+            '{\n  "version": 1,\n  "data": [\n    {\n      "number": 10000,\n'
+            '      "package_name": "com.example.widget"\n    }\n  ]\n}'
+        )
+
+    def test_trailing_default_is_not_part_of_the_value(self):
+        settings = parse_bugreport_settings()
+
+        record = find(settings, "namespace_one/streaming_blocked_components")[0]
+        assert record["value"] == "com.example.dialer,com.example.camera"
+        assert record["default"] == "com.android.settings,\n        com.android.vending"
+
+    def test_trailing_metadata_is_not_part_of_the_value(self):
+        settings = parse_bugreport_settings()
+
+        record = find(settings, "lock_screen_show_notifications")[0]
+        assert record["value"] == "1"
+        assert record["defaultSystemSet"] == "true"
+        assert record["isValuePreservedInRestore"] == "true"
+
+        # Without a default, the tag or the restore token follows the value.
+        record = find(settings, "accessibility_enabled")[1]
+        assert record["value"] == "0"
+        assert record["tag"] == "null"
+        assert "default" not in record
+
+        record = find(settings, "send_action_app_error")[0]
+        assert record["value"] == "1"
+        assert record["isValuePreservedInRestore"] == "false"
+
+    def test_dumps_after_the_last_block_are_not_part_of_the_last_row(self):
+        settings = parse_bugreport_settings()
+
+        assert settings.results[-1] == {
+            "namespace": "secure",
+            "user": "10",
+            "_id": "311",
+            "name": "accessibility_enabled",
+            "pkg": "android",
+            "value": "0",
+            "tag": "null",
+            "history": [],
+        }
+
+    def test_repeated_names_are_kept_as_separate_records(self):
+        settings = parse_bugreport_settings()
+
+        widgets = find(settings, "widget_instance_data")
+        assert [record["_id"] for record in widgets] == ["771", "41654"]
+
+        accessibility = find(settings, "accessibility_enabled")
+        assert [(record["user"], record["value"]) for record in accessibility] == [
+            ("0", "1"),
+            ("10", "0"),
+        ]
+
+    def test_setting_without_recording_package(self):
+        settings = parse_bugreport_settings()
+
+        record = find(settings, "hidden_api_blacklist_exemptions")[0]
+        assert "pkg" not in record
+        assert record["value"] == "{null}"
+
+    def test_history_timestamps_resolved_against_section_end(self):
+        settings = parse_bugreport_settings()
+
+        # The section was dumped on 2022-03-29, so an 11-02 entry belongs to
+        # the previous year and an 03-14 entry to the same year.
+        assert find(settings, "development_settings_enabled")[0]["history"] == [
+            {
+                "timestamp": "2021-11-02 11:21:22.212000",
+                "oldValue": "null",
+                "newValue": "1",
+                "pkg": "com.android.settings",
+            },
+            {
+                "timestamp": "2022-03-14 09:02:11.100000",
+                "oldValue": "1",
+                "newValue": "0",
+                "pkg": "com.example.updater",
+            },
+        ]
+
+    def test_history_without_a_section_end_has_no_timestamp(self):
+        settings = Settings()
+        settings.parse(
+            "SECURE SETTINGS (user 0)\n"
+            "_id:240 name:accessibility_enabled pkg:android value:1\n"
+            "\tHistory (accessibility_enabled)\n"
+            "\t\ttime:03-28 22:41:07.980 mode:update oldValue:0 newValue:1 "
+            "package:com.example.helper\n"
+        )
+
+        assert settings.results[0]["history"] == [
+            {
+                "timestamp": None,
+                "oldValue": "0",
+                "newValue": "1",
+                "pkg": "com.example.helper",
+            }
+        ]
+
+    def test_dangerous_setting_is_detected_with_the_changing_package(self):
+        settings = parse_bugreport_settings()
+        settings.check_indicators()
+
+        assert len(settings.alertstore.alerts) == 1
+        alert = settings.alertstore.alerts[0]
+        assert "accessibility_enabled = 1" in alert.message
+        assert alert.event_time == "2022-03-28 22:41:07.980000"
+        assert alert.event["history"][0]["pkg"] == "com.example.helper"

tests/android_androidqf/test_settings.py

@@ -6,27 +6,12 @@
 from pathlib import Path
 
 from mvt.android.modules.androidqf.aqf_settings import AQFSettings
-from mvt.android.artifacts.settings import Settings
 from mvt.common.module import run_module
 
 from ..utils import get_android_androidqf, list_files
 
 
 class TestSettingsModule:
-    def test_bugreport_settings_format(self):
-        settings = Settings()
-        settings.parse(
-            "GLOBAL SETTINGS (user 0)\n"
-            "_id:1 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true\n"
-            "SECURE SETTINGS (user 10)\n"
-            "_id:2 name:accessibility_enabled pkg:android value:1\n"
-        )
-
-        assert settings.results == {
-            "global:user_0": {"adb_wifi_enabled": "0"},
-            "secure:user_10": {"accessibility_enabled": "1"},
-        }
-
     def test_parsing(self):
         data_path = get_android_androidqf()
         m = AQFSettings(target_path=data_path)
@@ -34,7 +19,13 @@ class TestSettingsModule:
         parent_path = Path(data_path).absolute().parent.as_posix()
         m.from_dir(parent_path, files)
         run_module(m)
-        assert len(m.results) == 1
-        assert "random" in m.results.keys()
+        assert len(m.results) == 9
+        assert {result["namespace"] for result in m.results} == {"random"}
+        assert m.results[0] == {
+            "namespace": "random",
+            "user": None,
+            "name": "samsung_errorlog_agree",
+            "value": "0",
+        }
         assert len(m.alertstore.alerts) == 1
         assert "samsung_errorlog_agree" in m.alertstore.alerts[0].message

tests/android_bugreport/test_bugreport.py

@@ -10,6 +10,7 @@ from mvt.android.modules.bugreport.dumpsys_appops import DumpsysAppops
 from mvt.android.modules.bugreport.dumpsys_getprop import DumpsysGetProp
 from mvt.android.modules.bugreport.dumpsys_packages import DumpsysPackages
 from mvt.android.modules.bugreport.dumpsys_receivers import DumpsysReceivers
+from mvt.android.modules.bugreport.settings import Settings
 from mvt.android.modules.bugreport.tombstones import Tombstones
 from mvt.common.module import run_module
 
@@ -93,6 +94,25 @@ class TestBugreportAnalysis:
         assert alert.event == malicious_receiver
         assert alert.matched_indicator.value == "com.android.services"
 
+    def test_settings_module(self):
+        m = self.launch_bug_report_module(Settings)
+        assert len(m.results) == 12
+
+        assert len(m.alertstore.alerts) == 1
+        assert "accessibility_enabled = 1" in m.alertstore.alerts[0].message
+
+        assert len(m.timeline) == 3
+        change = [
+            entry
+            for entry in m.timeline
+            if entry["timestamp"] == "2022-03-28 22:41:07.980000"
+        ][0]
+        assert change["event"] == "settings_change"
+        assert change["data"] == (
+            'secure setting "accessibility_enabled" changed from "0" to "1" '
+            "by com.example.helper"
+        )
+
     def test_tombstones_modules(self):
         m = self.launch_bug_report_module(Tombstones)
         assert len(m.results) == 2

tests/artifacts/android_data/bugreport/dumpstate.txt

@@ -264,5 +264,53 @@ ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled)
 ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34)
 ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled)
 ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable)
+-------------------------------------------------------------------------------
+DUMP OF SERVICE settings:
+CONFIG SETTINGS (user 0)
+_id:682 name:namespace_one/blocked_components pkg:com.example.services value:com.android.settings,com.android.vending,
+com.example.dialer,
+com.example.camera default:com.android.settings,
+        com.android.vending,
+        com.example.dialer defaultSystemSet:false
+_id:684 name:namespace_one/streaming_blocked_components pkg:com.example.services value:com.example.dialer,com.example.camera default:com.android.settings,
+        com.android.vending defaultSystemSet:false
+_id:680 name:namespace_one/allowed_packages pkg:com.example.services value:com.example.messaging,
+com.example.chat
 
+GLOBAL SETTINGS (user 0)
+_id:2070 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true
+_id:778 name:hidden_api_blacklist_exemptions value:{null}
+_id:9640 name:send_action_app_error pkg:android value:1 notPreservedInRestore
+_id:9631 name:development_settings_enabled pkg:com.android.settings value:1 default:1 defaultSystemSet:true
+	History (development_settings_enabled)
+		time:11-02 11:21:22.212 mode:update oldValue:null newValue:1 package:com.android.settings
+		time:03-14 09:02:11.100 mode:update oldValue:1 newValue:0 package:com.example.updater
+_id:771 name:widget_instance_data pkg:com.android.systemui value:{
+  "version": 1,
+  "data": [
+    {
+      "number": 10000,
+      "package_name": "com.example.widget"
+    }
+  ]
+} defaultSystemSet:true
+_id:41654 name:widget_instance_data pkg:com.android.systemui value:{
+  "version": 3,
+  "data": []
+} defaultSystemSet:true
 
+SECURE SETTINGS (user 0)
+_id:907 name:lock_screen_show_notifications pkg:com.android.settings value:1 default:1 defaultSystemSet:true isValuePreservedInRestore:true
+_id:240 name:accessibility_enabled pkg:android value:1 default:0 defaultSystemSet:true
+	History (accessibility_enabled)
+		time:03-28 22:41:07.980 mode:update oldValue:0 newValue:1 package:com.example.helper
+
+SECURE SETTINGS (user 10)
+_id:311 name:accessibility_enabled pkg:android value:0 tag:null
+
+GENERATION REGISTRY
+Maximum number of backing stores:8
+Number of backing stores:1
+_Backing store for type:SETTINGS_SECURE user:10 size:1024 cachedEntries:1
+
+--------- 0.019s was the duration of dumpsys settings, ending at: 2022-03-29 23:14:28