Cache Android bugreport dumpstate sections

src/mvt/android/modules/bugreport/base.py

@@ -4,15 +4,36 @@
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 import datetime
 import fnmatch
+import io
 import logging
 import os
+import re
 from pathlib import Path
 from typing import List, Optional
 from zipfile import ZipFile
 
+from mvt.android.artifacts.artifact import AndroidArtifact
 from mvt.common.module import ModuleResults, MVTModule
 
 
+_DUMPSTATE_BYTES_CACHE_KEY = ("android_bugreport", "dumpstate", "bytes")
+_DUMPSTATE_SECTIONS_CACHE_KEY = ("android_bugreport", "dumpstate", "sections")
+_SYSTEM_PROPERTIES_SECTION = b"------ SYSTEM PROPERTIES"
+_DUMPSYS_SEPARATORS = {
+    b"DUMP OF SERVICE accessibility:",
+    b"DUMP OF SERVICE adb:",
+    b"DUMP OF SERVICE appops:",
+    b"DUMP OF SERVICE batterystats:",
+    b"DUMP OF SERVICE dbinfo:",
+    b"DUMP OF SERVICE package:",
+    b"DUMP OF SERVICE platform_compat:",
+}
+_DUMPSTATE_DELIMITER = (
+    b"------------------------------------------------------------------------------"
+)
+_GETPROP_LINE_RE = re.compile(rb"\[(.+?)\]: \[(.+?)\]")
+
+
 class BugReportModule(MVTModule):
     """This class provides a base for all Android Bug Report modules."""
 
@@ -73,8 +94,10 @@ class BugReportModule(MVTModule):
                 if not self.extract_path:
                     raise ValueError("extract_path is not set")
                 joined = os.path.join(self.extract_path, file_path)
-                if not Path(joined).resolve().is_relative_to(
-                    Path(self.extract_path).resolve()
+                if (
+                    not Path(joined)
+                    .resolve()
+                    .is_relative_to(Path(self.extract_path).resolve())
                 ):
                     raise ValueError("unsafe file_path")
                 handle = open(joined, "rb")
@@ -85,23 +108,134 @@ class BugReportModule(MVTModule):
             return data
 
     def _get_dumpstate_file(self) -> Optional[bytes]:
-        main = self._get_files_by_pattern("main_entry.txt")
-        if main:
-            main_content = self._get_file_content(main[0])
-            try:
-                return self._get_file_content(main_content.decode().strip())
-            except KeyError:
-                return None
+        with self.resource_lock:
+            if _DUMPSTATE_BYTES_CACHE_KEY in self.resource_cache:
+                return self.resource_cache[_DUMPSTATE_BYTES_CACHE_KEY]
 
-        dumpstate_logs = self._get_files_by_pattern("dumpState_*.log")
-        if dumpstate_logs:
-            return self._get_file_content(dumpstate_logs[0])
+            content = None
+            main = self._get_files_by_pattern("main_entry.txt")
+            if main:
+                main_content = self._get_file_content(main[0])
+                try:
+                    content = self._get_file_content(main_content.decode().strip())
+                except KeyError:
+                    pass
+            else:
+                dumpstate_logs = self._get_files_by_pattern("dumpState_*.log")
+                if dumpstate_logs:
+                    content = self._get_file_content(dumpstate_logs[0])
+                else:
+                    dumpsys_files = self._get_files_by_pattern("*/dumpsys.txt")
+                    if dumpsys_files:
+                        content = self._get_file_content(dumpsys_files[0])
 
-        dumpsys_files = self._get_files_by_pattern("*/dumpsys.txt")
-        if dumpsys_files:
-            return self._get_file_content(dumpsys_files[0])
+            self.resource_cache[_DUMPSTATE_BYTES_CACHE_KEY] = content
+            return content
 
-        return None
+    def _get_dumpstate_sections(self) -> dict[bytes, bytes]:
+        with self.resource_lock:
+            if _DUMPSTATE_SECTIONS_CACHE_KEY in self.resource_cache:
+                return self.resource_cache[_DUMPSTATE_SECTIONS_CACHE_KEY]
+
+            section_lines: dict[bytes, list[bytes]] = {
+                separator: [] for separator in _DUMPSYS_SEPARATORS
+            }
+            section_lines[_SYSTEM_PROPERTIES_SECTION] = []
+            current_section = None
+            completed_sections = set()
+            in_system_properties = False
+            content = self._get_dumpstate_file()
+
+            if content:
+                for raw_line in io.BytesIO(content):
+                    line = raw_line.rstrip(b"\r\n")
+                    stripped = line.strip()
+
+                    if stripped.startswith(_SYSTEM_PROPERTIES_SECTION):
+                        in_system_properties = True
+                    elif in_system_properties:
+                        if stripped == b"------":
+                            in_system_properties = False
+                        elif _GETPROP_LINE_RE.search(line):
+                            section_lines[_SYSTEM_PROPERTIES_SECTION].append(line)
+
+                    if stripped in _DUMPSYS_SEPARATORS:
+                        if current_section is not None:
+                            completed_sections.add(current_section)
+                        current_section = (
+                            None if stripped in completed_sections else stripped
+                        )
+                        continue
+
+                    if stripped.startswith(_DUMPSTATE_DELIMITER):
+                        if current_section is not None:
+                            completed_sections.add(current_section)
+                        current_section = None
+                        continue
+
+                    if current_section is not None:
+                        section_lines[current_section].append(line)
+
+            sections = {
+                separator: b"\n".join(lines)
+                for separator, lines in section_lines.items()
+            }
+            self.resource_cache[_DUMPSTATE_SECTIONS_CACHE_KEY] = sections
+            return sections
+
+    def _get_dumpsys_section_bytes(self, separator: bytes) -> Optional[bytes]:
+        sections = self._get_dumpstate_sections()
+        if separator in sections:
+            return sections[separator]
+
+        content = self._get_dumpstate_file()
+        if not content:
+            return None
+        return AndroidArtifact.extract_dumpsys_section(content, separator, binary=True)
+
+    def _get_dumpsys_section(
+        self, separator: str, errors: str = "replace"
+    ) -> Optional[str]:
+        cache_key = (
+            "android_bugreport",
+            "dumpstate",
+            "section",
+            separator,
+            errors,
+        )
+        with self.resource_lock:
+            if cache_key in self.resource_cache:
+                return self.resource_cache[cache_key]
+
+            section_bytes = self._get_dumpsys_section_bytes(separator.encode("utf-8"))
+            section = (
+                section_bytes.decode("utf-8", errors=errors)
+                if section_bytes is not None
+                else None
+            )
+            self.resource_cache[cache_key] = section
+            return section
+
+    def _get_system_properties_text(self, errors: str = "ignore") -> Optional[str]:
+        cache_key = (
+            "android_bugreport",
+            "dumpstate",
+            "system_properties",
+            errors,
+        )
+        with self.resource_lock:
+            if cache_key in self.resource_cache:
+                return self.resource_cache[cache_key]
+
+            sections = self._get_dumpstate_sections()
+            section_bytes = sections.get(_SYSTEM_PROPERTIES_SECTION)
+            section = (
+                section_bytes.decode("utf-8", errors=errors)
+                if section_bytes is not None
+                else None
+            )
+            self.resource_cache[cache_key] = section
+            return section
 
     def _get_file_modification_time(self, file_path: str) -> datetime.datetime:
         if self.zip_archive:

src/mvt/android/modules/bugreport/dumpsys_accessibility.py

@@ -34,18 +34,16 @@ class DumpsysAccessibility(DumpsysAccessibilityArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        full_dumpsys = self._get_dumpstate_file()
-        if not full_dumpsys:
+        content = self._get_dumpsys_section(
+            "DUMP OF SERVICE accessibility:", errors="ignore"
+        )
+        if content is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        content = self.extract_dumpsys_section(
-            full_dumpsys.decode("utf-8", errors="ignore"),
-            "DUMP OF SERVICE accessibility:",
-        )
         self.parse(content)
 
         for result in self.results:

src/mvt/android/modules/bugreport/dumpsys_activities.py

@@ -38,19 +38,14 @@ class DumpsysActivities(DumpsysPackageActivitiesArtifact, BugReportModule):
         self.results = results if results else []
 
     def run(self) -> None:
-        content = self._get_dumpstate_file()
-        if not content:
+        section = self._get_dumpsys_section("DUMP OF SERVICE package:", errors="ignore")
+        if section is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        # Extract package section
-        section = self.extract_dumpsys_section(
-            content.decode("utf-8", errors="ignore"), "DUMP OF SERVICE package:"
-        )
-
         # Parse
         self.parse(section)
 

src/mvt/android/modules/bugreport/dumpsys_adb_state.py

@@ -34,19 +34,14 @@ class DumpsysADBState(DumpsysADBArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        full_dumpsys = self._get_dumpstate_file()
-        if not full_dumpsys:
+        content = self._get_dumpsys_section_bytes(b"DUMP OF SERVICE adb:")
+        if content is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        content = self.extract_dumpsys_section(
-            full_dumpsys,
-            b"DUMP OF SERVICE adb:",
-            binary=True,
-        )
         self.parse(content)
         if self.results:
             self.log.info(

src/mvt/android/modules/bugreport/dumpsys_appops.py

@@ -34,17 +34,14 @@ class DumpsysAppops(DumpsysAppopsArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        content = self._get_dumpstate_file()
-        if not content:
+        section = self._get_dumpsys_section("DUMP OF SERVICE appops:")
+        if section is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        section = self.extract_dumpsys_section(
-            content.decode("utf-8", errors="replace"), "DUMP OF SERVICE appops:"
-        )
         self.parse(section)
 
         self.log.info(

src/mvt/android/modules/bugreport/dumpsys_battery_daily.py

@@ -34,17 +34,14 @@ class DumpsysBatteryDaily(DumpsysBatteryDailyArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        content = self._get_dumpstate_file()
-        if not content:
+        dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE batterystats:")
+        if dumpsys_section is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        dumpsys_section = self.extract_dumpsys_section(
-            content.decode("utf-8", errors="replace"), "DUMP OF SERVICE batterystats:"
-        )
         self.parse(dumpsys_section)
 
         self.log.info("Extracted a total of %d battery daily stats", len(self.results))

src/mvt/android/modules/bugreport/dumpsys_battery_history.py

@@ -34,17 +34,14 @@ class DumpsysBatteryHistory(DumpsysBatteryHistoryArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        content = self._get_dumpstate_file()
-        if not content:
+        dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE batterystats:")
+        if dumpsys_section is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        dumpsys_section = self.extract_dumpsys_section(
-            content.decode("utf-8", errors="replace"), "DUMP OF SERVICE batterystats:"
-        )
         self.parse(dumpsys_section)
 
         self.log.info(

src/mvt/android/modules/bugreport/dumpsys_dbinfo.py

@@ -36,18 +36,14 @@ class DumpsysDBInfo(DumpsysDBInfoArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        data = self._get_dumpstate_file()
-        if not data:
+        section = self._get_dumpsys_section("DUMP OF SERVICE dbinfo:", errors="ignore")
+        if section is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        section = self.extract_dumpsys_section(
-            data.decode("utf-8", errors="ignore"), "DUMP OF SERVICE dbinfo:"
-        )
-
         self.parse(section)
         self.log.info(
             "Extracted a total of %d database connection pool records",

src/mvt/android/modules/bugreport/dumpsys_getprop.py

@@ -36,29 +36,13 @@ class DumpsysGetProp(GetPropArtifact, BugReportModule):
         self.results = [] if not results else results
 
     def run(self) -> None:
-        content = self._get_dumpstate_file()
-        if not content:
+        content = self._get_system_properties_text()
+        if content is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        lines = []
-        in_getprop = False
-
-        for line in content.decode(errors="ignore").splitlines():
-            if line.strip().startswith("------ SYSTEM PROPERTIES"):
-                in_getprop = True
-                continue
-
-            if not in_getprop:
-                continue
-
-            if line.strip() == "------":
-                break
-
-            lines.append(line)
-
-        self.parse("\n".join(lines))
+        self.parse(content)
         self.log.info("Extracted %d Android system properties", len(self.results))

src/mvt/android/modules/bugreport/dumpsys_packages.py

@@ -35,17 +35,14 @@ class DumpsysPackages(DumpsysPackagesArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        data = self._get_dumpstate_file()
-        if not data:
+        content = self._get_dumpsys_section("DUMP OF SERVICE package:")
+        if content is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        content = self.extract_dumpsys_section(
-            data.decode("utf-8", errors="replace"), "DUMP OF SERVICE package:"
-        )
         self.parse(content)
 
         for result in self.results:

src/mvt/android/modules/bugreport/dumpsys_platform_compat.py

@@ -34,18 +34,14 @@ class DumpsysPlatformCompat(DumpsysPlatformCompatArtifact, BugReportModule):
         )
 
     def run(self) -> None:
-        data = self._get_dumpstate_file()
-        if not data:
+        content = self._get_dumpsys_section("DUMP OF SERVICE platform_compat:")
+        if content is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        decoded_data = data.decode("utf-8", errors="replace")
-        content = self.extract_dumpsys_section(
-            decoded_data, "DUMP OF SERVICE platform_compat:"
-        )
         self.parse(content)
 
         self.log.info("Found %d uninstalled apps", len(self.results))

src/mvt/android/modules/bugreport/dumpsys_receivers.py

@@ -36,17 +36,13 @@ class DumpsysReceivers(DumpsysReceiversArtifact, BugReportModule):
         self.results = results if results else {}
 
     def run(self) -> None:
-        content = self._get_dumpstate_file()
-        if not content:
+        dumpsys_section = self._get_dumpsys_section("DUMP OF SERVICE package:")
+        if dumpsys_section is None:
             self.log.error(
                 "Unable to find dumpstate file. "
                 "Did you provide a valid bug report archive?"
             )
             return
 
-        dumpsys_section = self.extract_dumpsys_section(
-            content.decode("utf-8", errors="replace"), "DUMP OF SERVICE package:"
-        )
-
         self.parse(dumpsys_section)
         self.log.info("Extracted receivers for %d intents", len(self.results))

src/mvt/common/command.py

@@ -70,6 +70,7 @@ class Command:
             raise ValueError("jobs must be at least 1")
         self.jobs = jobs
         self._resource_lock = threading.RLock()
+        self._resource_cache: dict[object, Any] = {}
 
         # This dictionary can contain options that will be passed down from
         # the Command to all modules. This can for example be used to pass
@@ -377,6 +378,7 @@ class Command:
                 for dependency in module.dependencies
             }
             m.resource_lock = self._resource_lock
+            m.resource_cache = self._resource_cache
 
             if self.iocs.total_ioc_count:
                 # IOC collections are shared read-only during module execution.

src/mvt/common/module.py

@@ -88,6 +88,9 @@ class MVTModule:
         # Commands replace this with a command-scoped lock for modules sharing
         # archive handles or other non-thread-safe read resources.
         self.resource_lock = threading.RLock()
+        # Commands also replace this with a command-scoped cache. Cached values
+        # must be immutable, because parallel modules can read them concurrently.
+        self.resource_cache: Dict[object, Any] = {}
 
     def get_dependency_results(
         self, module_class: type["MVTModule"]

tests/android_bugreport/test_bugreport.py

@@ -4,8 +4,13 @@
 #   https://license.mvt.re/1.1/
 
 import os
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from io import BytesIO
 from pathlib import Path
+from unittest.mock import MagicMock
 
+from mvt.android.modules.bugreport.base import BugReportModule
 from mvt.android.modules.bugreport.dumpsys_appops import DumpsysAppops
 from mvt.android.modules.bugreport.dumpsys_getprop import DumpsysGetProp
 from mvt.android.modules.bugreport.dumpsys_packages import DumpsysPackages
@@ -90,3 +95,53 @@ class TestBugreportAnalysis:
         m = self.launch_bug_report_module(Tombstones)
         assert len(m.results) == 2
         assert m.results[1]["pid"] == 3559
+
+    def test_dumpstate_section_is_cached_across_parallel_modules(self):
+        dumpstate = (
+            b"header\n"
+            b"------ SYSTEM PROPERTIES (getprop) ------\n"
+            b"[persist.sys.timezone]: [Europe/Berlin]\n"
+            b"------------------------------------------------------------------------------\n"
+            b"------\n"
+            b"DUMP OF SERVICE package:\n"
+            b"package data\n"
+            b"------------------------------------------------------------------------------\n"
+            b"DUMP OF SERVICE package:\n"
+            b"duplicate package data\n"
+            b"------------------------------------------------------------------------------\n"
+            b"DUMP OF SERVICE adb:\n"
+            b"adb data\n"
+            b"------------------------------------------------------------------------------\n"
+        )
+        archive = MagicMock()
+        archive.open.side_effect = lambda _: BytesIO(dumpstate)
+        shared_lock = threading.RLock()
+        shared_cache = {}
+        modules = []
+
+        for _ in range(4):
+            module = BugReportModule()
+            module.from_zip(archive, ["dumpState_test.log"])
+            module.resource_lock = shared_lock
+            module.resource_cache = shared_cache
+            modules.append(module)
+
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            sections = list(
+                executor.map(
+                    lambda module: module._get_dumpsys_section(
+                        "DUMP OF SERVICE package:"
+                    ),
+                    modules,
+                )
+            )
+
+        assert sections == ["package data"] * 4
+        assert all(section is sections[0] for section in sections)
+        assert modules[0]._get_dumpsys_section_bytes(b"DUMP OF SERVICE adb:") == (
+            b"adb data"
+        )
+        assert modules[0]._get_system_properties_text() == (
+            "[persist.sys.timezone]: [Europe/Berlin]"
+        )
+        archive.open.assert_called_once_with("dumpState_test.log")