From 0e231eefaf4b26dff35de02b5de2fe1cb91fd52c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donncha=20=C3=93=20Cearbhaill?= Date: Fri, 4 Sep 2026 17:44:51 +0200 Subject: [PATCH] Parse dumpsys settings as per-record results The `dumpsys settings` parser matched a single regex per line, which truncated every value that spans more than one line and, when `defaultSystemSet:` did not fall on the first line, left the trailing `default:` metadata inside the value. It also keyed results by setting name within a namespace, so a name recorded twice kept only the last row and a row without a `pkg:` field was dropped entirely. Replace it with a line loop that accumulates one record at a time and splits the `key:value` fields once the whole record has been read. Results become a list of records carrying the fields dumpsys prints: namespace, user, _id, name, value, pkg, default and defaultSystemSet, plus the per-setting change history. History timestamps are printed without a year, so they are resolved against the "ending at:" time of the section and serialized into the timeline. This shows which package changed a security-relevant setting, and when. The androidqf settings module shares this artifact, so it now emits the same record shape. --- src/mvt/android/artifacts/settings.py | 264 +++++++++++++++--- .../android/modules/androidqf/aqf_settings.py | 46 +-- src/mvt/android/modules/bugreport/settings.py | 3 +- tests/android/test_artifact_settings.py | 138 +++++++++ tests/android_androidqf/test_settings.py | 25 +- tests/android_bugreport/test_bugreport.py | 20 ++ .../android_data/bugreport/dumpstate.txt | 42 +++ 7 files changed, 448 insertions(+), 90 deletions(-) create mode 100644 tests/android/test_artifact_settings.py diff --git a/src/mvt/android/artifacts/settings.py b/src/mvt/android/artifacts/settings.py index e0cd5f0b..878165a5 100644 --- a/src/mvt/android/artifacts/settings.py +++ b/src/mvt/android/artifacts/settings.py @@ -4,6 +4,11 @@ # https://license.mvt.re/1.1/ import re +from datetime import datetime +from typing import Optional, Sequence + +from mvt.common.module_types import ModuleAtomicResult, ModuleSerializedResult +from mvt.common.utils import convert_datetime_to_iso from .artifact import AndroidArtifact @@ -60,45 +65,232 @@ ANDROID_DANGEROUS_SETTINGS = [ }, ] +# dumpsys prints the fields of a setting record, and of a change history entry, +# always in this order and separated by a single space. +SETTING_FIELDS = ("_id", "name", "pkg", "value") +HISTORY_FIELDS = ("time", "mode", "oldValue", "newValue", "package") + +NAMESPACE_PATTERN = re.compile( + r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$" +) +SECTION_END_PATTERN = re.compile(r"ending at: (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})") + class Settings(AndroidArtifact): - def parse(self, content: str) -> None: - self.results: dict[str, dict[str, str]] = {} - namespace: str | None = None - for line in content.splitlines(): - heading = re.match( - r"^(CONFIG|GLOBAL|SECURE|SYSTEM) SETTINGS \(user (\d+)\)$", - line.strip(), - ) - if heading: - namespace = f"{heading.group(1).lower()}:user_{heading.group(2)}" - self.results[namespace] = {} + """Parser for the `dumpsys settings` output. + + Every row of the settings provider becomes one result, keeping the fields + dumpsys prints alongside the value: the row id, the package which recorded + the setting, the default, and the change history. A setting name can appear + more than once within a namespace, so results are a list rather than a + mapping. + """ + + def serialize(self, result: ModuleAtomicResult) -> ModuleSerializedResult: + records = [] + for entry in result.get("history", []): + if not entry.get("timestamp"): continue - if namespace is None or not line.startswith("_id:"): - continue - setting = re.match( - r"^_id:\S+\s+name:(.*?)\s+pkg:.*?\s+value:(.*?)" - r"(?:\s+default:.*\s+defaultSystemSet:(?:true|false))?$", - line, + + records.append( + { + "timestamp": entry["timestamp"], + "module": self.__class__.__name__, + "event": "settings_change", + "data": ( + f"{result.get('namespace')} setting " + f'"{result.get("name")}" changed from ' + f'"{entry.get("oldValue")}" to "{entry.get("newValue")}" ' + f"by {entry.get('pkg')}" + ), + } ) - if setting: - self.results[namespace][setting.group(1)] = setting.group(2) + + return records def check_indicators(self) -> None: - for namespace, settings in self.results.items(): - for key, value in settings.items(): - for danger in ANDROID_DANGEROUS_SETTINGS: - # Check if one of the dangerous settings is using an unsafe - # value (different than the one specified). - if danger["key"] == key and danger["safe_value"] != value: - self.alertstore.medium( - f'Found suspicious "{namespace}" setting "{key} = {value}" ({danger["description"]})', - "", - { - "namespace": namespace, - "key": key, - "value": value, - "description": danger["description"], - }, - ) - break + for result in self.results: + name = result.get("name") + value = result.get("value") + for danger in ANDROID_DANGEROUS_SETTINGS: + # Check if one of the dangerous settings is using an unsafe + # value (different than the one specified). + if danger["key"] != name or danger["safe_value"] == value: + continue + + history = result.get("history") or [] + self.alertstore.medium( + f'Found suspicious "{result.get("namespace")}" setting ' + f'"{name} = {value}" ({danger["description"]})', + history[-1]["timestamp"] if history else "", + result, + ) + break + + def parse(self, content: str) -> None: + self.results: list[ModuleAtomicResult] = [] + section_end = self._parse_section_end(content) + namespace: Optional[str] = None + user: Optional[str] = None + record_lines: list[str] = [] + history_lines: list[str] = [] + in_history = False + + def flush() -> None: + nonlocal record_lines, history_lines, in_history + if record_lines: + self.results.append( + self._build_record( + namespace, user, record_lines, history_lines, section_end + ) + ) + record_lines = [] + history_lines = [] + in_history = False + + for line in content.splitlines(): + heading = NAMESPACE_PATTERN.match(line.strip()) + if heading: + flush() + namespace = heading.group(1).lower() + user = heading.group(2) + continue + + if line.startswith("--------- "): + # dumpsys closes every section with a duration trailer. + flush() + namespace = None + continue + + if namespace is None: + continue + + if line.startswith("_id:"): + flush() + record_lines = [line] + continue + + if not record_lines: + continue + + stripped = line.strip() + if stripped.startswith("History ("): + in_history = True + continue + + if in_history: + if stripped.startswith("time:"): + history_lines.append(stripped) + elif stripped and history_lines: + # A history entry can be wrapped over several lines. + history_lines[-1] += " " + stripped + continue + + # Anything else continues the value of the record being read. + record_lines.append(line) + + flush() + + @staticmethod + def _split_fields(text: str, keys: Sequence[str]) -> dict[str, str]: + """Split the `key:value` fields of one record. + + Values are free-form and may contain spaces and newlines, so a field + runs up to the start of the next key which is actually present. Keys + dumpsys did not print are skipped. + """ + fields: dict[str, str] = {} + key = keys[0] + if not text.startswith(f"{key}:"): + return fields + + remainder = text[len(key) + 1 :] + for next_key in keys[1:]: + value, separator, rest = remainder.partition(f" {next_key}:") + if separator: + fields[key] = value + key, remainder = next_key, rest + + fields[key] = remainder + return fields + + @staticmethod + def _parse_section_end(content: str) -> Optional[datetime]: + """Return the time the settings section was dumped, if reported.""" + match = SECTION_END_PATTERN.search(content) + if not match: + return None + + try: + return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S") + except ValueError: + return None + + @staticmethod + def _resolve_timestamp( + value: str, section_end: Optional[datetime] + ) -> Optional[str]: + """Add the missing year to a `MM-DD HH:MM:SS.mmm` history timestamp. + + dumpsys prints the change history without a year, so it is resolved + against the time the section was dumped: the most recent matching date + at or before that time. + """ + if section_end is None: + return None + + try: + partial = datetime.strptime(value, "%m-%d %H:%M:%S.%f") + timestamp = partial.replace(year=section_end.year) + if timestamp > section_end: + timestamp = partial.replace(year=section_end.year - 1) + except ValueError: + return None + + return convert_datetime_to_iso(timestamp) + + def _parse_history( + self, line: str, section_end: Optional[datetime] + ) -> ModuleAtomicResult: + fields = self._split_fields(line, HISTORY_FIELDS) + return { + "timestamp": self._resolve_timestamp(fields.get("time", ""), section_end), + "oldValue": fields.get("oldValue"), + "newValue": fields.get("newValue"), + "pkg": fields.get("package"), + } + + def _build_record( + self, + namespace: Optional[str], + user: Optional[str], + record_lines: list[str], + history_lines: list[str], + section_end: Optional[datetime], + ) -> ModuleAtomicResult: + text = "\n".join(record_lines).rstrip() + + # `default:` and `defaultSystemSet:` are printed after the value, and + # the default may itself be multi-line, so peel them off the end first. + default = None + default_system_set = None + head, separator, tail = text.rpartition(" defaultSystemSet:") + if separator: + default_system_set = tail.strip() + text = head + head, separator, tail = text.rpartition(" default:") + if separator: + default = tail + text = head + + record: ModuleAtomicResult = {"namespace": namespace, "user": user} + record.update(self._split_fields(text, SETTING_FIELDS)) + if default is not None: + record["default"] = default + if default_system_set is not None: + record["defaultSystemSet"] = default_system_set + + record["history"] = [ + self._parse_history(entry, section_end) for entry in history_lines + ] + return record diff --git a/src/mvt/android/modules/androidqf/aqf_settings.py b/src/mvt/android/modules/androidqf/aqf_settings.py index 8d5bb518..397e121f 100644 --- a/src/mvt/android/modules/androidqf/aqf_settings.py +++ b/src/mvt/android/modules/androidqf/aqf_settings.py @@ -3,11 +3,7 @@ # Use of this software is governed by the MVT License 1.1 that can be found at # https://license.mvt.re/1.1/ -import logging -from typing import Optional - from mvt.android.artifacts.settings import Settings as SettingsArtifact -from mvt.common.module_types import ModuleResults from .base import AndroidQFModule @@ -15,43 +11,23 @@ from .base import AndroidQFModule class AQFSettings(SettingsArtifact, AndroidQFModule): """This module analyse setting files""" - def __init__( - self, - file_path: Optional[str] = None, - target_path: Optional[str] = None, - results_path: Optional[str] = None, - module_options: Optional[dict] = None, - log: logging.Logger = logging.getLogger(__name__), - results: Optional[ModuleResults] = None, - ) -> None: - super().__init__( - file_path=file_path, - target_path=target_path, - results_path=results_path, - module_options=module_options, - log=log, - results=results, - ) - self.results: dict = results if results is not None else {} - def run(self) -> None: for setting_file in self._get_files_by_pattern("*/settings_*.txt"): namespace = setting_file[setting_file.rfind("_") + 1 : -4] - self.results[namespace] = {} data = self._get_file_content(setting_file) for line in data.decode("utf-8").splitlines(): - line = line.strip() - try: - key, value = line.split("=", 1) - except ValueError: + name, separator, value = line.strip().partition("=") + if not separator: continue - try: - self.results[namespace][key] = value - except IndexError: - continue + self.results.append( + { + "namespace": namespace, + "user": None, + "name": name, + "value": value, + } + ) - self.log.info( - "Identified %d settings", sum([len(val) for val in self.results.values()]) - ) + self.log.info("Identified %d settings", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/settings.py b/src/mvt/android/modules/bugreport/settings.py index 20180adf..6df77a65 100644 --- a/src/mvt/android/modules/bugreport/settings.py +++ b/src/mvt/android/modules/bugreport/settings.py @@ -18,5 +18,4 @@ class Settings(SettingsArtifact, BugReportModule): data.decode("utf-8", errors="replace"), "DUMP OF SERVICE settings:" ) self.parse(section) - count = sum(len(settings) for settings in self.results.values()) - self.log.info("Identified %d Android settings", count) + self.log.info("Identified %d Android settings", len(self.results)) diff --git a/tests/android/test_artifact_settings.py b/tests/android/test_artifact_settings.py new file mode 100644 index 00000000..d3f10521 --- /dev/null +++ b/tests/android/test_artifact_settings.py @@ -0,0 +1,138 @@ +# Mobile Verification Toolkit (MVT) +# Copyright (c) 2021-2023 The MVT Authors. +# Use of this software is governed by the MVT License 1.1 that can be found at +# https://license.mvt.re/1.1/ + +from mvt.android.artifacts.settings import Settings + +from ..utils import get_artifact + + +def parse_bugreport_settings() -> Settings: + settings = Settings() + with open(get_artifact("android_data/bugreport/dumpstate.txt")) as handle: + data = handle.read() + + settings.parse(settings.extract_dumpsys_section(data, "DUMP OF SERVICE settings:")) + return settings + + +def find(settings: Settings, name: str) -> list: + return [result for result in settings.results if result["name"] == name] + + +class TestSettingsArtifact: + def test_parsing(self): + settings = parse_bugreport_settings() + + assert len(settings.results) == 11 + assert {result["namespace"] for result in settings.results} == { + "config", + "global", + "secure", + } + assert settings.results[0] == { + "namespace": "config", + "user": "0", + "_id": "682", + "name": "namespace_one/blocked_components", + "pkg": "com.example.services", + "value": ( + "com.android.settings,com.android.vending,\n" + "com.example.dialer,\n" + "com.example.camera" + ), + "default": ( + "com.android.settings,\n" + " com.android.vending,\n" + " com.example.dialer" + ), + "defaultSystemSet": "false", + "history": [], + } + + def test_multiline_values_are_kept_whole(self): + settings = parse_bugreport_settings() + + assert find(settings, "namespace_one/allowed_packages")[0]["value"] == ( + "com.example.messaging,\ncom.example.chat" + ) + assert find(settings, "widget_instance_data")[0]["value"] == ( + '{\n "version": 1,\n "data": [\n {\n "number": 10000,\n' + ' "package_name": "com.example.widget"\n }\n ]\n}' + ) + + def test_trailing_default_is_not_part_of_the_value(self): + settings = parse_bugreport_settings() + + record = find(settings, "namespace_one/streaming_blocked_components")[0] + assert record["value"] == "com.example.dialer,com.example.camera" + assert record["default"] == "com.android.settings,\n com.android.vending" + + def test_repeated_names_are_kept_as_separate_records(self): + settings = parse_bugreport_settings() + + widgets = find(settings, "widget_instance_data") + assert [record["_id"] for record in widgets] == ["771", "41654"] + + accessibility = find(settings, "accessibility_enabled") + assert [(record["user"], record["value"]) for record in accessibility] == [ + ("0", "1"), + ("10", "0"), + ] + + def test_setting_without_recording_package(self): + settings = parse_bugreport_settings() + + record = find(settings, "hidden_api_blacklist_exemptions")[0] + assert "pkg" not in record + assert record["value"] == "{null}" + + def test_history_timestamps_resolved_against_section_end(self): + settings = parse_bugreport_settings() + + # The section was dumped on 2022-03-29, so an 11-02 entry belongs to + # the previous year and an 03-14 entry to the same year. + assert find(settings, "development_settings_enabled")[0]["history"] == [ + { + "timestamp": "2021-11-02 11:21:22.212000", + "oldValue": "null", + "newValue": "1", + "pkg": "com.android.settings", + }, + { + "timestamp": "2022-03-14 09:02:11.100000", + "oldValue": "1", + "newValue": "0", + "pkg": "com.example.updater", + }, + ] + + def test_history_without_a_section_end_has_no_timestamp(self): + settings = Settings() + settings.parse( + "SECURE SETTINGS (user 0)\n" + "_id:240 name:accessibility_enabled pkg:android value:1\n" + "\tHistory (accessibility_enabled)\n" + "\t\ttime:03-28 22:41:07.980 mode:update oldValue:0 newValue:1 " + "package:com.example.helper\n" + ) + + assert settings.results[0]["history"] == [ + { + "timestamp": None, + "oldValue": "0", + "newValue": "1", + "pkg": "com.example.helper", + } + ] + + def test_dangerous_setting_is_detected_with_the_changing_package(self): + settings = parse_bugreport_settings() + settings.check_indicators() + + assert len(settings.alertstore.alerts) == 1 + alert = settings.alertstore.alerts[0] + assert "accessibility_enabled = 1" in alert.message + assert alert.event_time == "2022-03-28 22:41:07.980000" + assert alert.event["history"][0]["pkg"] == "com.example.helper" diff --git a/tests/android_androidqf/test_settings.py b/tests/android_androidqf/test_settings.py index 6edfec81..3391565c 100644 --- a/tests/android_androidqf/test_settings.py +++ b/tests/android_androidqf/test_settings.py @@ -6,27 +6,12 @@ from pathlib import Path from mvt.android.modules.androidqf.aqf_settings import AQFSettings -from mvt.android.artifacts.settings import Settings from mvt.common.module import run_module from ..utils import get_android_androidqf, list_files class TestSettingsModule: - def test_bugreport_settings_format(self): - settings = Settings() - settings.parse( - "GLOBAL SETTINGS (user 0)\n" - "_id:1 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true\n" - "SECURE SETTINGS (user 10)\n" - "_id:2 name:accessibility_enabled pkg:android value:1\n" - ) - - assert settings.results == { - "global:user_0": {"adb_wifi_enabled": "0"}, - "secure:user_10": {"accessibility_enabled": "1"}, - } - def test_parsing(self): data_path = get_android_androidqf() m = AQFSettings(target_path=data_path) @@ -34,7 +19,13 @@ class TestSettingsModule: parent_path = Path(data_path).absolute().parent.as_posix() m.from_dir(parent_path, files) run_module(m) - assert len(m.results) == 1 - assert "random" in m.results.keys() + assert len(m.results) == 9 + assert {result["namespace"] for result in m.results} == {"random"} + assert m.results[0] == { + "namespace": "random", + "user": None, + "name": "samsung_errorlog_agree", + "value": "0", + } assert len(m.alertstore.alerts) == 1 assert "samsung_errorlog_agree" in m.alertstore.alerts[0].message diff --git a/tests/android_bugreport/test_bugreport.py b/tests/android_bugreport/test_bugreport.py index 413bb180..e0a92f7a 100644 --- a/tests/android_bugreport/test_bugreport.py +++ b/tests/android_bugreport/test_bugreport.py @@ -10,6 +10,7 @@ from mvt.android.modules.bugreport.dumpsys_appops import DumpsysAppops from mvt.android.modules.bugreport.dumpsys_getprop import DumpsysGetProp from mvt.android.modules.bugreport.dumpsys_packages import DumpsysPackages from mvt.android.modules.bugreport.dumpsys_receivers import DumpsysReceivers +from mvt.android.modules.bugreport.settings import Settings from mvt.android.modules.bugreport.tombstones import Tombstones from mvt.common.module import run_module @@ -93,6 +94,25 @@ class TestBugreportAnalysis: assert alert.event == malicious_receiver assert alert.matched_indicator.value == "com.android.services" + def test_settings_module(self): + m = self.launch_bug_report_module(Settings) + assert len(m.results) == 11 + + assert len(m.alertstore.alerts) == 1 + assert "accessibility_enabled = 1" in m.alertstore.alerts[0].message + + assert len(m.timeline) == 3 + change = [ + entry + for entry in m.timeline + if entry["timestamp"] == "2022-03-28 22:41:07.980000" + ][0] + assert change["event"] == "settings_change" + assert change["data"] == ( + 'secure setting "accessibility_enabled" changed from "0" to "1" ' + "by com.example.helper" + ) + def test_tombstones_modules(self): m = self.launch_bug_report_module(Tombstones) assert len(m.results) == 2 diff --git a/tests/artifacts/android_data/bugreport/dumpstate.txt b/tests/artifacts/android_data/bugreport/dumpstate.txt index c888ed5c..34479e9f 100644 --- a/tests/artifacts/android_data/bugreport/dumpstate.txt +++ b/tests/artifacts/android_data/bugreport/dumpstate.txt @@ -264,5 +264,47 @@ ChangeId(143539591; name=SELINUX_LATEST_CHANGES; disabled) ChangeId(247079863; name=DISALLOW_INVALID_GROUP_REFERENCE; enableSinceTargetSdk=34) ChangeId(174227820; name=FORCE_DISABLE_HEVC_SUPPORT; disabled) ChangeId(168419799; name=DOWNSCALED; disabled; packageOverrides={com.google.android.apps.tachyon=false, org.torproject.torbrowser=false}; rawOverrides={org.torproject.torbrowser=false, org.article19.circulo.next=false}; overridable) +------------------------------------------------------------------------------- +DUMP OF SERVICE settings: +CONFIG SETTINGS (user 0) +_id:682 name:namespace_one/blocked_components pkg:com.example.services value:com.android.settings,com.android.vending, +com.example.dialer, +com.example.camera default:com.android.settings, + com.android.vending, + com.example.dialer defaultSystemSet:false +_id:684 name:namespace_one/streaming_blocked_components pkg:com.example.services value:com.example.dialer,com.example.camera default:com.android.settings, + com.android.vending defaultSystemSet:false +_id:680 name:namespace_one/allowed_packages pkg:com.example.services value:com.example.messaging, +com.example.chat +GLOBAL SETTINGS (user 0) +_id:2070 name:adb_wifi_enabled pkg:android value:0 default:0 defaultSystemSet:true +_id:778 name:hidden_api_blacklist_exemptions value:{null} +_id:9631 name:development_settings_enabled pkg:com.android.settings value:1 default:1 defaultSystemSet:true + History (development_settings_enabled) + time:11-02 11:21:22.212 mode:update oldValue:null newValue:1 package:com.android.settings + time:03-14 09:02:11.100 mode:update oldValue:1 newValue:0 package:com.example.updater +_id:771 name:widget_instance_data pkg:com.android.systemui value:{ + "version": 1, + "data": [ + { + "number": 10000, + "package_name": "com.example.widget" + } + ] +} defaultSystemSet:true +_id:41654 name:widget_instance_data pkg:com.android.systemui value:{ + "version": 3, + "data": [] +} defaultSystemSet:true +SECURE SETTINGS (user 0) +_id:907 name:lock_screen_show_notifications pkg:com.android.settings value:1 default:1 defaultSystemSet:true +_id:240 name:accessibility_enabled pkg:android value:1 default:0 defaultSystemSet:true + History (accessibility_enabled) + time:03-28 22:41:07.980 mode:update oldValue:0 newValue:1 package:com.example.helper + +SECURE SETTINGS (user 10) +_id:311 name:accessibility_enabled pkg:android value:0 default:0 defaultSystemSet:true + +--------- 0.019s was the duration of dumpsys settings, ending at: 2022-03-29 23:14:28