From 361bc72a4eea07a3752d3c6d30dec949ac6ca8f7 Mon Sep 17 00:00:00 2001 From: Janik Besendorf Date: Wed, 15 Jul 2026 23:51:53 +0200 Subject: [PATCH] Cache Android bugreport dumpstate sections --- src/mvt/android/modules/bugreport/base.py | 166 ++++++++++++++++-- .../bugreport/dumpsys_accessibility.py | 10 +- .../modules/bugreport/dumpsys_activities.py | 9 +- .../modules/bugreport/dumpsys_adb_state.py | 9 +- .../modules/bugreport/dumpsys_appops.py | 7 +- .../bugreport/dumpsys_battery_daily.py | 7 +- .../bugreport/dumpsys_battery_history.py | 7 +- .../modules/bugreport/dumpsys_dbinfo.py | 8 +- .../modules/bugreport/dumpsys_getprop.py | 22 +-- .../modules/bugreport/dumpsys_packages.py | 7 +- .../bugreport/dumpsys_platform_compat.py | 8 +- .../modules/bugreport/dumpsys_receivers.py | 8 +- src/mvt/common/command.py | 2 + src/mvt/common/module.py | 3 + tests/android_bugreport/test_bugreport.py | 55 ++++++ 15 files changed, 235 insertions(+), 93 deletions(-) diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index ddd69e3..2d49fe5 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -4,15 +4,36 @@ # https://github.com/mvt-project/mvt/blob/main/LICENSE import datetime import fnmatch +import io import logging import os +import re from pathlib import Path from typing import List, Optional from zipfile import ZipFile +from mvt.android.artifacts.artifact import AndroidArtifact from mvt.common.module import ModuleResults, MVTModule +_DUMPSTATE_BYTES_CACHE_KEY = ("android_bugreport", "dumpstate", "bytes") +_DUMPSTATE_SECTIONS_CACHE_KEY = ("android_bugreport", "dumpstate", "sections") +_SYSTEM_PROPERTIES_SECTION = b"------ SYSTEM PROPERTIES" +_DUMPSYS_SEPARATORS = { + b"DUMP OF SERVICE accessibility:", + b"DUMP OF SERVICE adb:", + b"DUMP OF SERVICE appops:", + b"DUMP OF SERVICE batterystats:", + b"DUMP OF SERVICE dbinfo:", + b"DUMP OF SERVICE package:", + b"DUMP OF SERVICE platform_compat:", +} +_DUMPSTATE_DELIMITER = ( + b"------------------------------------------------------------------------------" +) +_GETPROP_LINE_RE = re.compile(rb"\[(.+?)\]: \[(.+?)\]") + + class BugReportModule(MVTModule): """This class provides a base for all Android Bug Report modules.""" @@ -73,8 +94,10 @@ class BugReportModule(MVTModule): if not self.extract_path: raise ValueError("extract_path is not set") joined = os.path.join(self.extract_path, file_path) - if not Path(joined).resolve().is_relative_to( - Path(self.extract_path).resolve() + if ( + not Path(joined) + .resolve() + .is_relative_to(Path(self.extract_path).resolve()) ): raise ValueError("unsafe file_path") handle = open(joined, "rb") @@ -85,23 +108,134 @@ class BugReportModule(MVTModule): return data def _get_dumpstate_file(self) -> Optional[bytes]: - main = self._get_files_by_pattern("main_entry.txt") - if main: - main_content = self._get_file_content(main[0]) - try: - return self._get_file_content(main_content.decode().strip()) - except KeyError: - return None + with self.resource_lock: + if _DUMPSTATE_BYTES_CACHE_KEY in self.resource_cache: + return self.resource_cache[_DUMPSTATE_BYTES_CACHE_KEY] - dumpstate_logs = self._get_files_by_pattern("dumpState_*.log") - if dumpstate_logs: - return self._get_file_content(dumpstate_logs[0]) + content = None + main = self._get_files_by_pattern("main_entry.txt") + if main: + main_content = self._get_file_content(main[0]) + try: + content = self._get_file_content(main_content.decode().strip()) + except KeyError: + pass + else: + dumpstate_logs = self._get_files_by_pattern("dumpState_*.log") + if dumpstate_logs: + content = self._get_file_content(dumpstate_logs[0]) + else: + dumpsys_files = self._get_files_by_pattern("*/dumpsys.txt") + if dumpsys_files: + content = self._get_file_content(dumpsys_files[0]) - dumpsys_files = self._get_files_by_pattern("*/dumpsys.txt") - if dumpsys_files: - return self._get_file_content(dumpsys_files[0]) + self.resource_cache[_DUMPSTATE_BYTES_CACHE_KEY] = content + return content - return None + def _get_dumpstate_sections(self) -> dict[bytes, bytes]: + with self.resource_lock: + if _DUMPSTATE_SECTIONS_CACHE_KEY in self.resource_cache: + return self.resource_cache[_DUMPSTATE_SECTIONS_CACHE_KEY] + + section_lines: dict[bytes, list[bytes]] = { + separator: [] for separator in _DUMPSYS_SEPARATORS + } + section_lines[_SYSTEM_PROPERTIES_SECTION] = [] + current_section = None + completed_sections = set() + in_system_properties = False + content = self._get_dumpstate_file() + + if content: + for raw_line in io.BytesIO(content): + line = raw_line.rstrip(b"\r\n") + stripped = line.strip() + + if stripped.startswith(_SYSTEM_PROPERTIES_SECTION): + in_system_properties = True + elif in_system_properties: + if stripped == b"------": + in_system_properties = False + elif _GETPROP_LINE_RE.search(line): + section_lines[_SYSTEM_PROPERTIES_SECTION].append(line) + + if stripped in _DUMPSYS_SEPARATORS: + if current_section is not None: + completed_sections.add(current_section) + current_section = ( + None if stripped in completed_sections else stripped + ) + continue + + if stripped.startswith(_DUMPSTATE_DELIMITER): + if current_section is not None: + completed_sections.add(current_section) + current_section = None + continue + + if current_section is not None: + section_lines[current_section].append(line) + + sections = { + separator: b"\n".join(lines) + for separator, lines in section_lines.items() + } + self.resource_cache[_DUMPSTATE_SECTIONS_CACHE_KEY] = sections + return sections + + def _get_dumpsys_section_bytes(self, separator: bytes) -> Optional[bytes]: + sections = self._get_dumpstate_sections() + if separator in sections: + return sections[separator] + + content = self._get_dumpstate_file() + if not content: + return None + return AndroidArtifact.extract_dumpsys_section(content, separator, binary=True) + + def _get_dumpsys_section( + self, separator: str, errors: str = "replace" + ) -> Optional[str]: + cache_key = ( + "android_bugreport", + "dumpstate", + "section", + separator, + errors, + ) + with self.resource_lock: + if cache_key in self.resource_cache: + return self.resource_cache[cache_key] + + section_bytes = self._get_dumpsys_section_bytes(separator.encode("utf-8")) + section = ( + section_bytes.decode("utf-8", errors=errors) + if section_bytes is not None + else None + ) + self.resource_cache[cache_key] = section + return section + + def _get_system_properties_text(self, errors: str = "ignore") -> Optional[str]: + cache_key = ( + "android_bugreport", + "dumpstate", + "system_properties", + errors, + ) + with self.resource_lock: + if cache_key in self.resource_cache: + return self.resource_cache[cache_key] + + sections = self._get_dumpstate_sections() + section_bytes = sections.get(_SYSTEM_PROPERTIES_SECTION) + section = ( + section_bytes.decode("utf-8", errors=errors) + if section_bytes is not None + else None + ) + self.resource_cache[cache_key] = section + return section def _get_file_modification_time(self, file_path: str) -> datetime.datetime: if self.zip_archive: diff --git a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py index 72208c0..09df4e3 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py +++ b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py @@ -34,18 +34,16 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule): ) def run(self) -> None: - full_dumpsys = self._get_dumpstate_file() - if not full_dumpsys: + content = self._get_dumpsys_section( + "DUMP OF SERVICE accessibility:", errors="ignore" + ) + if content is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - content = self.extract_dumpsys_section( - full_dumpsys.decode("utf-8", errors="ignore"), - "DUMP OF SERVICE accessibility:", - ) self.parse(content) for result in self.results: diff --git a/src/mvt/android/modules/bugreport/dumpsys_activities.py b/src/mvt/android/modules/bugreport/dumpsys_activities.py index 2800e95..b5959af 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_activities.py +++ b/src/mvt/android/modules/bugreport/dumpsys_activities.py @@ -38,19 +38,14 @@ class DumpsysActivities(DumpsysPackageActivitiesArtifact, BugReportModule): self.results = results if results else [] def run(self) -> None: - content = self._get_dumpstate_file() - if not content: + section = self._get_dumpsys_section("DUMP OF SERVICE package:", errors="ignore") + if section is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - # Extract package section - section = self.extract_dumpsys_section( - content.decode("utf-8", errors="ignore"), "DUMP OF SERVICE package:" - ) - # Parse self.parse(section) diff --git a/src/mvt/android/modules/bugreport/dumpsys_adb_state.py b/src/mvt/android/modules/bugreport/dumpsys_adb_state.py index 506af30..7b9cb5c 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_adb_state.py +++ b/src/mvt/android/modules/bugreport/dumpsys_adb_state.py @@ -34,19 +34,14 @@ class DumpsysADBState(DumpsysADBArtifact, BugReportModule): ) def run(self) -> None: - full_dumpsys = self._get_dumpstate_file() - if not full_dumpsys: + content = self._get_dumpsys_section_bytes(b"DUMP OF SERVICE adb:") + if content is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - content = self.extract_dumpsys_section( - full_dumpsys, - b"DUMP OF SERVICE adb:", - binary=True, - ) self.parse(content) if self.results: self.log.info( diff --git a/src/mvt/android/modules/bugreport/dumpsys_appops.py b/src/mvt/android/modules/bugreport/dumpsys_appops.py index 91122cb..8edef12 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_appops.py +++ b/src/mvt/android/modules/bugreport/dumpsys_appops.py @@ -34,17 +34,14 @@ class DumpsysAppops(DumpsysAppopsArtifact, BugReportModule): ) def run(self) -> None: - content = self._get_dumpstate_file() - if not content: + section = self._get_dumpsys_section("DUMP OF SERVICE appops:") + if section is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - section = self.extract_dumpsys_section( - content.decode("utf-8", errors="replace"), "DUMP OF SERVICE appops:" - ) self.parse(section) self.log.info( diff --git a/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py b/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py index a7c0c72..5a55bec 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py +++ b/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py @@ -34,17 +34,14 @@ class DumpsysBatteryDaily(DumpsysBatteryDailyArtifact, BugReportModule): ) def run(self) -> None: - content = self._get_dumpstate_file() - if not content: + dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE batterystats:") + if dumpsys_section is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - dumpsys_section = self.extract_dumpsys_section( - content.decode("utf-8", errors="replace"), "DUMP OF SERVICE batterystats:" - ) self.parse(dumpsys_section) self.log.info("Extracted a total of %d battery daily stats", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/dumpsys_battery_history.py b/src/mvt/android/modules/bugreport/dumpsys_battery_history.py index 42d395d..236e9b7 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_battery_history.py +++ b/src/mvt/android/modules/bugreport/dumpsys_battery_history.py @@ -34,17 +34,14 @@ class DumpsysBatteryHistory(DumpsysBatteryHistoryArtifact, BugReportModule): ) def run(self) -> None: - content = self._get_dumpstate_file() - if not content: + dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE batterystats:") + if dumpsys_section is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - dumpsys_section = self.extract_dumpsys_section( - content.decode("utf-8", errors="replace"), "DUMP OF SERVICE batterystats:" - ) self.parse(dumpsys_section) self.log.info( diff --git a/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py b/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py index 13ba8b3..c8f573a 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py +++ b/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py @@ -36,18 +36,14 @@ class DumpsysDBInfo(DumpsysDBInfoArtifact, BugReportModule): ) def run(self) -> None: - data = self._get_dumpstate_file() - if not data: + section = self._get_dumpsys_section("DUMP OF SERVICE dbinfo:", errors="ignore") + if section is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - section = self.extract_dumpsys_section( - data.decode("utf-8", errors="ignore"), "DUMP OF SERVICE dbinfo:" - ) - self.parse(section) self.log.info( "Extracted a total of %d database connection pool records", diff --git a/src/mvt/android/modules/bugreport/dumpsys_getprop.py b/src/mvt/android/modules/bugreport/dumpsys_getprop.py index 198a1d2..1b54fd0 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_getprop.py +++ b/src/mvt/android/modules/bugreport/dumpsys_getprop.py @@ -36,29 +36,13 @@ class DumpsysGetProp(GetPropArtifact, BugReportModule): self.results = [] if not results else results def run(self) -> None: - content = self._get_dumpstate_file() - if not content: + content = self._get_system_properties_text() + if content is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - lines = [] - in_getprop = False - - for line in content.decode(errors="ignore").splitlines(): - if line.strip().startswith("------ SYSTEM PROPERTIES"): - in_getprop = True - continue - - if not in_getprop: - continue - - if line.strip() == "------": - break - - lines.append(line) - - self.parse("\n".join(lines)) + self.parse(content) self.log.info("Extracted %d Android system properties", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/dumpsys_packages.py b/src/mvt/android/modules/bugreport/dumpsys_packages.py index aebec4c..186327f 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_packages.py +++ b/src/mvt/android/modules/bugreport/dumpsys_packages.py @@ -35,17 +35,14 @@ class DumpsysPackages(DumpsysPackagesArtifact, BugReportModule): ) def run(self) -> None: - data = self._get_dumpstate_file() - if not data: + content = self._get_dumpsys_section("DUMP OF SERVICE package:") + if content is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - content = self.extract_dumpsys_section( - data.decode("utf-8", errors="replace"), "DUMP OF SERVICE package:" - ) self.parse(content) for result in self.results: diff --git a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py index 968bc25..030e859 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py +++ b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py @@ -34,18 +34,14 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule): ) def run(self) -> None: - data = self._get_dumpstate_file() - if not data: + content = self._get_dumpsys_section("DUMP OF SERVICE platform_compat:") + if content is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - decoded_data = data.decode("utf-8", errors="replace") - content = self.extract_dumpsys_section( - decoded_data, "DUMP OF SERVICE platform_compat:" - ) self.parse(content) self.log.info("Found %d uninstalled apps", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/dumpsys_receivers.py b/src/mvt/android/modules/bugreport/dumpsys_receivers.py index 8907d5e..ef53f5f 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_receivers.py +++ b/src/mvt/android/modules/bugreport/dumpsys_receivers.py @@ -36,17 +36,13 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule): self.results = results if results else {} def run(self) -> None: - content = self._get_dumpstate_file() - if not content: + dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE package:") + if dumpsys_section is None: self.log.error( "Unable to find dumpstate file. " "Did you provide a valid bug report archive?" ) return - dumpsys_section = self.extract_dumpsys_section( - content.decode("utf-8", errors="replace"), "DUMP OF SERVICE package:" - ) - self.parse(dumpsys_section) self.log.info("Extracted receivers for %d intents", len(self.results)) diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index acfadd3..6938271 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -70,6 +70,7 @@ class Command: raise ValueError("jobs must be at least 1") self.jobs = jobs self._resource_lock = threading.RLock() + self._resource_cache: dict[object, Any] = {} # This dictionary can contain options that will be passed down from # the Command to all modules. This can for example be used to pass @@ -377,6 +378,7 @@ class Command: for dependency in module.dependencies } m.resource_lock = self._resource_lock + m.resource_cache = self._resource_cache if self.iocs.total_ioc_count: # IOC collections are shared read-only during module execution. diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index e597f15..e7a7e1c 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -88,6 +88,9 @@ class MVTModule: # Commands replace this with a command-scoped lock for modules sharing # archive handles or other non-thread-safe read resources. self.resource_lock = threading.RLock() + # Commands also replace this with a command-scoped cache. Cached values + # must be immutable, because parallel modules can read them concurrently. + self.resource_cache: Dict[object, Any] = {} def get_dependency_results( self, module_class: type["MVTModule"] diff --git a/tests/android_bugreport/test_bugreport.py b/tests/android_bugreport/test_bugreport.py index 2bfaef5..220d3e9 100644 --- a/tests/android_bugreport/test_bugreport.py +++ b/tests/android_bugreport/test_bugreport.py @@ -4,8 +4,13 @@ # https://license.mvt.re/1.1/ import os +import threading +from concurrent.futures import ThreadPoolExecutor +from io import BytesIO from pathlib import Path +from unittest.mock import MagicMock +from mvt.android.modules.bugreport.base import BugReportModule from mvt.android.modules.bugreport.dumpsys_appops import DumpsysAppops from mvt.android.modules.bugreport.dumpsys_getprop import DumpsysGetProp from mvt.android.modules.bugreport.dumpsys_packages import DumpsysPackages @@ -90,3 +95,53 @@ class TestBugreportAnalysis: m = self.launch_bug_report_module(Tombstones) assert len(m.results) == 2 assert m.results[1]["pid"] == 3559 + + def test_dumpstate_section_is_cached_across_parallel_modules(self): + dumpstate = ( + b"header\n" + b"------ SYSTEM PROPERTIES (getprop) ------\n" + b"[persist.sys.timezone]: [Europe/Berlin]\n" + b"------------------------------------------------------------------------------\n" + b"------\n" + b"DUMP OF SERVICE package:\n" + b"package data\n" + b"------------------------------------------------------------------------------\n" + b"DUMP OF SERVICE package:\n" + b"duplicate package data\n" + b"------------------------------------------------------------------------------\n" + b"DUMP OF SERVICE adb:\n" + b"adb data\n" + b"------------------------------------------------------------------------------\n" + ) + archive = MagicMock() + archive.open.side_effect = lambda _: BytesIO(dumpstate) + shared_lock = threading.RLock() + shared_cache = {} + modules = [] + + for _ in range(4): + module = BugReportModule() + module.from_zip(archive, ["dumpState_test.log"]) + module.resource_lock = shared_lock + module.resource_cache = shared_cache + modules.append(module) + + with ThreadPoolExecutor(max_workers=4) as executor: + sections = list( + executor.map( + lambda module: module._get_dumpsys_section( + "DUMP OF SERVICE package:" + ), + modules, + ) + ) + + assert sections == ["package data"] * 4 + assert all(section is sections[0] for section in sections) + assert modules[0]._get_dumpsys_section_bytes(b"DUMP OF SERVICE adb:") == ( + b"adb data" + ) + assert modules[0]._get_system_properties_text() == ( + "[persist.sys.timezone]: [Europe/Berlin]" + ) + archive.open.assert_called_once_with("dumpState_test.log")